ALLThe encryption code. Read it yourself.
What ALL sends to ChatGPT travels end-to-end encrypted between your computer and ChatGPT. This is the code that does it. It is public, and anyone can audit it.
- Made on your computerThe key is created on your computer, not on ALL’s server.
- The server cannot open itALL’s server only passes encrypted data along. It has no way to open it.
- Anyone can audit itThe code below is the one published at tag v2.0.0. Compare the commit with the one on GitHub.
Loading the code… If it does not load, read it on GitHub.
Security center Who sees what Privacy View on GitHub ↗
Every file, as plain text
This is the same code as the explorer above, written out so it can be read without scripts. Tag v2.0.0, commit 6606d11e0733.
Files under files/
contracts
schemas
- acme-abandon-request.schema.json
- acme-budget-response.schema.json
- acme-confirm-request.schema.json
- acme-confirm-response.schema.json
- acme-dns-request.schema.json
- acme-dns-response.schema.json
- acme-expected-response.schema.json
- acme-lapsed-response.schema.json
- acme-lift-request.schema.json
- acme-lift-response.schema.json
- acme-release-request.schema.json
- acme-reserve-request.schema.json
- acme-reserve-response.schema.json
- authorize-request.schema.json
- authorize-response.schema.json
- cert-status-response.schema.json
- common.schema.json
- enroll-request.schema.json
- enroll-response.schema.json
- error-response.schema.json
- feed-response.schema.json
- fleet-e2e-block.schema.json
- jwks.schema.json
- node-assertion-claims.schema.json
- node-keys.schema.json
- ok-body.schema.json
- paywall.schema.json
- proof-claims.schema.json
- reconnect-state.schema.json
- selfcheck-result.schema.json
- session-claims.schema.json
- statement-claims.schema.json
- surfaces-file.schema.json
- wire-v2.grammar.json
- e2e_contracts.py
- fake_node.py
- policy.json
scripts
tests
tunnel
- .gitignore
- acme.go
- auth.go
- backend_prod.go
- build.sh
- certreload.go
- certreload_test.go
- clock.go
- clock_test.go
- config.go
- config_test.go
- contracts_test.go
- contracts_verify_test.go
- controlplane.go
- controlplane_test.go
- doc.go
- drain.go
- drain_test.go
- fakecp_test.go
- feed.go
- feed_test.go
- gated.go
- gated_test.go
- go.mod
- hardening_helpers_test.go
- helpers_test.go
- hints.go
- hints_test.go
- jsonparse.go
- jwks.go
- jwks_test.go
- limits.go
- limits_test.go
- loghygiene_test.go
- main.go
- metrics.go
- metrics_test.go
- nftables_test.go
- nodeproc_test.go
- observer.go
- observer_test.go
- openai_prefixes.go
- openai_prefixes_test.go
- outage_test.go
- policy.go
- policy.json
- proto.go
- protocol_test.go
- pump_buffers_test.go
- selfcheck.go
- selfcheck_test.go
- server.go
- server_test.go
- sni.go
- sni_test.go
- splice_test.go
- stale_keys_grace_test.go
- statement_test.go
- storm_test.go
- unauth_reserve_test.go
MANIFEST.json 952 lines · 28.9 KB
{
"allowlist": [
"contracts/policy.json",
"contracts/e2e_contracts.py",
"contracts/fake_node.py",
"contracts/schemas/**",
"contracts/vectors/**",
"contracts/ts/verify.ts",
"contracts/ts/package.json",
"src/all_mcp/e2e_cert.py",
"src/all_mcp/e2e_tunnel.py",
"src/all_mcp/e2e_terminator.py",
"src/all_mcp/tls_policy.py",
"src/all_mcp/e2e_ct_watch.py",
"src/all_mcp/e2e_policy.py",
"src/all_mcp/e2e_policy.json",
"src/all_mcp/tls_store.py",
"src/all_mcp/surfaces.py",
"src/all_mcp/tray/encryption_verify.py",
"scripts/e2e_pcap.py",
"tests/_pcap_builder.py",
"tunnel/**"
],
"count": 127,
"denylist": [
"tunnel/tap_test_hook.go",
"tunnel/tap_test_hook_test.go",
"tunnel/blindmutant_test_hook.go",
"tunnel/testnode.go",
"tunnel/testpeer.go",
"tunnel/testpeer_peek_unix.go",
"tunnel/testpeer_peek_windows.go",
"tunnel/backend_loopcp.go",
"tunnel/cmd/loadgen/**"
],
"files": [
{
"bytes": 41881,
"lines": 855,
"notice": "inspection-only",
"path": "contracts/e2e_contracts.py",
"sha256": "73f61a04183c0b6703085cb847361c1448e5172384d0d1b197bcb498f54a635d"
},
{
"bytes": 20231,
"lines": 371,
"notice": "inspection-only",
"path": "contracts/fake_node.py",
"sha256": "7c596a2d1f21a210bca7bed36cf16893686df0a55b8d84478077577e42710cc7"
},
{
"bytes": 12859,
"lines": 376,
"notice": "inspection-only",
"path": "contracts/policy.json",
"sha256": "2b8640396d0fa3f9787670b6a821d5d03ebf74c1eeb192ced726ecc741e18ac1"
},
{
"bytes": 909,
"lines": 12,
"notice": "inspection-only",
"path": "contracts/schemas/acme-abandon-request.schema.json",
"sha256": "8063101ceecc04dc1ea2b9e23174f5a57db0afd1fb00b4a91e351da0742c5c1b"
},
{
"bytes": 1901,
"lines": 38,
"notice": "inspection-only",
"path": "contracts/schemas/acme-budget-response.schema.json",
"sha256": "c6ce62d65fd2a505526a8d1f68ca25a369831e4e46cf95d7e1e7bc9a4fa4b72a"
},
{
"bytes": 726,
"lines": 15,
"notice": "inspection-only",
"path": "contracts/schemas/acme-confirm-request.schema.json",
"sha256": "df429f2a55528772a60673026de4240d645391ee5c7d3ea9a563c764ff6c6b6b"
},
{
"bytes": 428,
"lines": 13,
"notice": "inspection-only",
"path": "contracts/schemas/acme-confirm-response.schema.json",
"sha256": "ec1c2a283c0d37e6d796582df96f4f12e577512bfa88b93387a5fd0b1247abd4"
},
{
"bytes": 1091,
"lines": 28,
"notice": "inspection-only",
"path": "contracts/schemas/acme-dns-request.schema.json",
"sha256": "1ecd86fc392142ec24b70e936d4b88098c7d5e5a67f0cc94f329491fd1231564"
},
{
"bytes": 522,
"lines": 15,
"notice": "inspection-only",
"path": "contracts/schemas/acme-dns-response.schema.json",
"sha256": "576ee1acbc5353f1e07a68762cb6dbaf81821993700cc77fac0bf3abb82c315e"
},
{
"bytes": 2027,
"lines": 33,
"notice": "inspection-only",
"path": "contracts/schemas/acme-expected-response.schema.json",
"sha256": "94e84646c11f30305859cdd1497ded5d954c0bf99276cb07aedaecbdc93166c5"
},
{
"bytes": 1606,
"lines": 29,
"notice": "inspection-only",
"path": "contracts/schemas/acme-lapsed-response.schema.json",
"sha256": "07d5a336bf4840d0b2fb2aa6465499cc3c5f45710d8c1efd0ce2d86cfe1954ac"
},
{
"bytes": 872,
"lines": 12,
"notice": "inspection-only",
"path": "contracts/schemas/acme-lift-request.schema.json",
"sha256": "c6e5534a91e67efbd7277276b2cdea5174544e249844f4a9d549489546522a2e"
},
{
"bytes": 545,
"lines": 14,
"notice": "inspection-only",
"path": "contracts/schemas/acme-lift-response.schema.json",
"sha256": "6cf0471730df9e80ab4cf1b909026dc50b73a7af881aae6040ecc4ae647b24e6"
},
{
"bytes": 536,
"lines": 12,
"notice": "inspection-only",
"path": "contracts/schemas/acme-release-request.schema.json",
"sha256": "45d7d4865e9ca939214a81eac0b1517b155fa91e540cf67a4c0c1a52c328b8d3"
},
{
"bytes": 739,
"lines": 13,
"notice": "inspection-only",
"path": "contracts/schemas/acme-reserve-request.schema.json",
"sha256": "750b4144d50c2d95fab6a14428be15aba8ed06d2a2f91f7ff3bacfcb0becb010"
},
{
"bytes": 1429,
"lines": 33,
"notice": "inspection-only",
"path": "contracts/schemas/acme-reserve-response.schema.json",
"sha256": "9ae646d6db9678c311658388f601e3acf8cad6199b20e16caa17c379ca099f7c"
},
{
"bytes": 779,
"lines": 15,
"notice": "inspection-only",
"path": "contracts/schemas/authorize-request.schema.json",
"sha256": "4a1cb8f258395fafad92eaca6c6536135ae571e91913e4639a89034db0d8f888"
},
{
"bytes": 1434,
"lines": 25,
"notice": "inspection-only",
"path": "contracts/schemas/authorize-response.schema.json",
"sha256": "7b0203bee339a538e4f3d18a7c861740ef98497611c6ba5d9555a68e8e547017"
},
{
"bytes": 1833,
"lines": 46,
"notice": "inspection-only",
"path": "contracts/schemas/cert-status-response.schema.json",
"sha256": "0227d33a6cf0bb048da0c7ed6ee78f7bd942da526e6c480e9d26a2de6dba5648"
},
{
"bytes": 4480,
"lines": 208,
"notice": "inspection-only",
"path": "contracts/schemas/common.schema.json",
"sha256": "ed88637eaedd2a79c02ce0b00cbd789061660929c045477e1faf687c95786fdf"
},
{
"bytes": 600,
"lines": 12,
"notice": "inspection-only",
"path": "contracts/schemas/enroll-request.schema.json",
"sha256": "a1a0e12f3baef786c8b12c536d5adb8b78ea6f587ddc1e8c672591089ba47807"
},
{
"bytes": 2220,
"lines": 25,
"notice": "inspection-only",
"path": "contracts/schemas/enroll-response.schema.json",
"sha256": "e2e8f1b48931125e46e0bdb96e53eeb25745afb375541c735960b93fae5d12d5"
},
{
"bytes": 748,
"lines": 15,
"notice": "inspection-only",
"path": "contracts/schemas/error-response.schema.json",
"sha256": "0c52fca010e135765ac70fb46ce808f3a23da92c27b6b29f436b489c66cecb2f"
},
{
"bytes": 1534,
"lines": 31,
"notice": "inspection-only",
"path": "contracts/schemas/feed-response.schema.json",
"sha256": "9188360b7dbde15e2399b1a8c26c44ccc7055231ca80fb9097042a343a611f55"
},
{
"bytes": 2350,
"lines": 25,
"notice": "inspection-only",
"path": "contracts/schemas/fleet-e2e-block.schema.json",
"sha256": "75b683d653ae2eb61946c65f4b9097647a30f16f4e06de3302e3f330ea688264"
},
{
"bytes": 1126,
"lines": 27,
"notice": "inspection-only",
"path": "contracts/schemas/jwks.schema.json",
"sha256": "d9fd9680af0758976a4b139e6466107a100b3e820dbfb4d68c42c3a74874a038"
},
{
"bytes": 1425,
"lines": 23,
"notice": "inspection-only",
"path": "contracts/schemas/node-assertion-claims.schema.json",
"sha256": "1754c179b0733e3ba46b5da78efb1f07e7f1a99cf012369de06f8ece04002e0f"
},
{
"bytes": 934,
"lines": 25,
"notice": "inspection-only",
"path": "contracts/schemas/node-keys.schema.json",
"sha256": "a8f19592bb6a9b759fa04c3ea98572cf49df7a80713a4dfdad1aa38fbff02be4"
},
{
"bytes": 1581,
"lines": 28,
"notice": "inspection-only",
"path": "contracts/schemas/ok-body.schema.json",
"sha256": "a872a18bfa912983f4e22d69d360ed0a2b53c34b34e699ab56f0a2623d577aaf"
},
{
"bytes": 2021,
"lines": 38,
"notice": "inspection-only",
"path": "contracts/schemas/paywall.schema.json",
"sha256": "145105e8d3c665ef132f361720340a30069167dd317b27c24e8904d36f5ff8cf"
},
{
"bytes": 1163,
"lines": 17,
"notice": "inspection-only",
"path": "contracts/schemas/proof-claims.schema.json",
"sha256": "3891174c7ce9985114faf2e38c6e6458b17a7c128e313ff5d6160cec6936ae31"
},
{
"bytes": 1124,
"lines": 13,
"notice": "inspection-only",
"path": "contracts/schemas/reconnect-state.schema.json",
"sha256": "aeaf428ae2c5bf2c4d63ab9877c9fd9162337f620e43ebc518ffcdbb2b0ce4db"
},
{
"bytes": 1486,
"lines": 17,
"notice": "inspection-only",
"path": "contracts/schemas/selfcheck-result.schema.json",
"sha256": "5df35f5c07a658a3e2699499f5a599286ad133ebe1f9e1fd796b42462307e049"
},
{
"bytes": 1355,
"lines": 24,
"notice": "inspection-only",
"path": "contracts/schemas/session-claims.schema.json",
"sha256": "2d85a806b2fd98ade32c6076336b8a5cdc6ec097abb5e25b977e52296a59b509"
},
{
"bytes": 2070,
"lines": 32,
"notice": "inspection-only",
"path": "contracts/schemas/statement-claims.schema.json",
"sha256": "51aebdb33bfd73c7cacd5e5d417dfc9b60c4150067e280b63763c5fb450d3dc6"
},
{
"bytes": 923,
"lines": 16,
"notice": "inspection-only",
"path": "contracts/schemas/surfaces-file.schema.json",
"sha256": "8222b1c680d4c14c492602ab56595ca86baf12aa835f9a4bed04a083adb5e590"
},
{
"bytes": 2961,
"lines": 44,
"notice": "inspection-only",
"path": "contracts/schemas/wire-v2.grammar.json",
"sha256": "a9b34e541beab21de6a30d5d153b9eed64f7ada0c99d1669b82b50530e65fb13"
},
{
"bytes": 42,
"lines": 4,
"notice": "inspection-only",
"path": "contracts/ts/package.json",
"sha256": "609158e6c5fbc237939fa3ddf7faab80ab690bdc0c8d584414a885130103c4e8"
},
{
"bytes": 39127,
"lines": 773,
"notice": "inspection-only",
"path": "contracts/ts/verify.ts",
"sha256": "0ec03a86377cfa815a7bcf5c7faaddb06c80b3b35001447f96efae1249a1374a"
},
{
"bytes": 70366,
"lines": 191,
"notice": "inspection-only",
"path": "contracts/vectors/bodies.json",
"sha256": "a3e36854d129f608857dbb36ac29e54b4950b6d3da4743cffbaeacae850624f9"
},
{
"bytes": 3471,
"lines": 114,
"notice": "inspection-only",
"path": "contracts/vectors/keys.json",
"sha256": "4a3b9d97864a59b1ed4d8108ab51c0c6fc04691b24011dd1a045915aa5737866"
},
{
"bytes": 4662,
"lines": 55,
"notice": "inspection-only",
"path": "contracts/vectors/labels.json",
"sha256": "a4cee8d171e82ac208ae10b0a6b003a34d41dfc690779935db9e8de57cca4690"
},
{
"bytes": 116563,
"lines": 155,
"notice": "inspection-only",
"path": "contracts/vectors/lines.json",
"sha256": "97bd3fd86dd98e69c9dfa83ee655349234878edfb72fc8db53821a1791763430"
},
{
"bytes": 190772,
"lines": 185,
"notice": "inspection-only",
"path": "contracts/vectors/tokens.json",
"sha256": "04feb86fbbbcd48115a665b0813e453ed3ba93326fe349878d33e796a47b6611"
},
{
"bytes": 273184,
"lines": 55,
"notice": "inspection-only",
"path": "contracts/vectors/transcripts.json",
"sha256": "9664674b26788bf860262657286cceb49c6134b80f2983024259521ef5676f9f"
},
{
"bytes": 42051,
"lines": 814,
"notice": "inspection-only",
"path": "scripts/e2e_pcap.py",
"sha256": "6244692d101b3c5851e84830e06f361eff87bac6b8716fc7d65a07d6836ccf9c"
},
{
"bytes": 84856,
"lines": 1530,
"notice": "inspection-only",
"path": "src/all_mcp/e2e_cert.py",
"sha256": "8626594491f9433fb48af59a6045ac632bfeafaa5d6949f16ccf8c2b296e3ba9"
},
{
"bytes": 31089,
"lines": 554,
"notice": "inspection-only",
"path": "src/all_mcp/e2e_ct_watch.py",
"sha256": "3b4d4b4cd61a575805bf7243b7ceb11cf8c421f2490a30ba6afcbfede1746de0"
},
{
"bytes": 6630,
"lines": 261,
"notice": "inspection-only",
"path": "src/all_mcp/e2e_policy.json",
"sha256": "43e8d978071df7f7303aa2bbcbbf67eb41e9235553266873cca86dcdd8c0b206"
},
{
"bytes": 1898,
"lines": 43,
"notice": "inspection-only",
"path": "src/all_mcp/e2e_policy.py",
"sha256": "87b9c654892a90327033d06d1f3044e597599d48d7cda7c7ae8b9a8a41704d7d"
},
{
"bytes": 43192,
"lines": 839,
"notice": "inspection-only",
"path": "src/all_mcp/e2e_terminator.py",
"sha256": "1c155ffd6a5fb34ce06b896469d222d2105cee18adc5d0131fc685409dd09129"
},
{
"bytes": 54123,
"lines": 1005,
"notice": "inspection-only",
"path": "src/all_mcp/e2e_tunnel.py",
"sha256": "d23fa82ccfe63ec04d3fd523ee69fcec78b549a40f63fc70761077cf1816847e"
},
{
"bytes": 47569,
"lines": 865,
"notice": "inspection-only",
"path": "src/all_mcp/surfaces.py",
"sha256": "edfd470d4b4b4c3903946f963ef7e5bb3df8e5d46cb3e12056633a502280369d"
},
{
"bytes": 52510,
"lines": 985,
"notice": "inspection-only",
"path": "src/all_mcp/tls_policy.py",
"sha256": "9278201c4df320f7666978ddb7741d44dfde15d3177dbcc329dd6472675ec7c0"
},
{
"bytes": 30620,
"lines": 660,
"notice": "inspection-only",
"path": "src/all_mcp/tls_store.py",
"sha256": "f46ccf6a098f9063567d4e3cbe3fb2b2dc7d7f02782607b14c4db61440883c21"
},
{
"bytes": 9360,
"lines": 143,
"notice": "inspection-only",
"path": "src/all_mcp/tray/encryption_verify.py",
"sha256": "641d44389dd58a6c0fe58c87787efe55aec6a62a6ceed367c9f5e2a1b0ec14b2"
},
{
"bytes": 5373,
"lines": 118,
"notice": "inspection-only",
"path": "tests/_pcap_builder.py",
"sha256": "c564d6b8de8ada455bf87980a56574c9e77b289faa70942042818c98c02e33c5"
},
{
"bytes": 118,
"lines": 4,
"notice": "inspection-only",
"path": "tunnel/.gitignore",
"sha256": "55cbf72f5a2cd5a6ed43b4e12a2802b0ab3e12fee9994753c892fd2130bddfc2"
},
{
"bytes": 3931,
"lines": 137,
"notice": "inspection-only",
"path": "tunnel/acme.go",
"sha256": "3304512e46dc44685a0ce96cc5106706616b889ee2c9cf7fcffa19a1de82266e"
},
{
"bytes": 17252,
"lines": 479,
"notice": "inspection-only",
"path": "tunnel/auth.go",
"sha256": "cc86de6463bd448aabdaf0c247685b1a359f7776c7ffab011df87d1eff46bb70"
},
{
"bytes": 2986,
"lines": 71,
"notice": "inspection-only",
"path": "tunnel/backend_prod.go",
"sha256": "96afc5f517b01869f7ce3f6d3e4c94e842911a59442a7d760d10290c418a4012"
},
{
"bytes": 4052,
"lines": 74,
"notice": "inspection-only",
"path": "tunnel/build.sh",
"sha256": "00d8faf414e8afc0ca55babf577f13bfca0118d37eb8f4c282f32eb8901a05a2"
},
{
"bytes": 6421,
"lines": 192,
"notice": "inspection-only",
"path": "tunnel/certreload.go",
"sha256": "cfec06e0b21e854b6b1f65e95f0976635653f73ac0ccb1284cf706467310a78f"
},
{
"bytes": 11102,
"lines": 297,
"notice": "inspection-only",
"path": "tunnel/certreload_test.go",
"sha256": "e0256e8bd580e5cf9308902f738dfe46914f1d9c15a87bb4707cb892c67f67c5"
},
{
"bytes": 2130,
"lines": 67,
"notice": "inspection-only",
"path": "tunnel/clock.go",
"sha256": "6856c8ce26b9bddc79576f98c8fa597e4609a3c1f070f3089ff3eb9e2892ad8e"
},
{
"bytes": 6128,
"lines": 200,
"notice": "inspection-only",
"path": "tunnel/clock_test.go",
"sha256": "c1eb5209c548b4daa2d28a31ec19b32da4118f8ab55d6850da13cf6f77d5c346"
},
{
"bytes": 15554,
"lines": 263,
"notice": "inspection-only",
"path": "tunnel/config.go",
"sha256": "5bf61e7144345dd41dfa0e7432deb34fad579363f24faeb28a1501a30ed8e4e4"
},
{
"bytes": 15821,
"lines": 410,
"notice": "inspection-only",
"path": "tunnel/config_test.go",
"sha256": "bc1a8173937b6ab7e718b187b553c76f6e3820f221cd794e24f46a92852fc394"
},
{
"bytes": 14084,
"lines": 378,
"notice": "inspection-only",
"path": "tunnel/contracts_test.go",
"sha256": "8b9de12e4326163907501aeb883220d87f579558753280db5b0843dd8efafa23"
},
{
"bytes": 48650,
"lines": 1643,
"notice": "inspection-only",
"path": "tunnel/contracts_verify_test.go",
"sha256": "85de26d2e6e520850a4a1bc4343dbae43e89bdbac289c6a20b09997e2feaff35"
},
{
"bytes": 19614,
"lines": 496,
"notice": "inspection-only",
"path": "tunnel/controlplane.go",
"sha256": "71e410e4670e3c4b11e2cefa7c84ab5edb9faf2f7cbe0514e6bc6f67741a32a4"
},
{
"bytes": 20099,
"lines": 514,
"notice": "inspection-only",
"path": "tunnel/controlplane_test.go",
"sha256": "ed538062f2ad979ee9b7dd66bd29e453ea47c9fb564e361b7e86647ecf15026f"
},
{
"bytes": 1681,
"lines": 27,
"notice": "inspection-only",
"path": "tunnel/deploy/all-tunnel-cert-check",
"sha256": "56bab5b37b5ae8bd8e97a98ed404e72f62801c113d6385fae5726e912a274f5a"
},
{
"bytes": 3896,
"lines": 64,
"notice": "inspection-only",
"path": "tunnel/deploy/all-tunnel-cert-hook",
"sha256": "cb7c4603c06c393075eb42be31018d0c3a9fa1be5ef5fcc62de49924f9bdf554"
},
{
"bytes": 2034,
"lines": 34,
"notice": "inspection-only",
"path": "tunnel/deploy/all-tunnel-cert-lib.sh",
"sha256": "b83c0f4c66dcd634aad2e6518f9ae190156c24f558f79c9f14ac61cf3dda09ef"
},
{
"bytes": 1182,
"lines": 14,
"notice": "inspection-only",
"path": "tunnel/deploy/all-tunnel-certbot-failed.service",
"sha256": "609fdf754ab3dd758e55d80138d79d349d7d1b05928eecf5600fd2556d690f74"
},
{
"bytes": 1707,
"lines": 25,
"notice": "inspection-only",
"path": "tunnel/deploy/all-tunnel-certbot.service",
"sha256": "4a6cec2694261e6f80a01db9f4c739d8b63e2dfdbbfb8d3fe584fa4f961b9928"
},
{
"bytes": 597,
"lines": 14,
"notice": "inspection-only",
"path": "tunnel/deploy/all-tunnel-certbot.timer",
"sha256": "471012448b1b422b3ac055effcfaa0e56a519a00cc8cdb570ff81645045d9e5e"
},
{
"bytes": 3754,
"lines": 67,
"notice": "inspection-only",
"path": "tunnel/deploy/all-tunnel.service",
"sha256": "59226542384e86f30faf66d0fe3066a2211818cc1f044d9c24cb350a702ecd14"
},
{
"bytes": 625,
"lines": 12,
"notice": "inspection-only",
"path": "tunnel/deploy/journald-volatile.conf",
"sha256": "795499b78c47641779f2ded3ca2fc5cdf61d01a00c61b8f85b2eccabfebf69cd"
},
{
"bytes": 4092,
"lines": 60,
"notice": "inspection-only",
"path": "tunnel/deploy/nftables.conf",
"sha256": "66732e2c01189dca08d44d90be8f2aa63c8255143d9a704202dc4bd19d6b62ae"
},
{
"bytes": 794,
"lines": 11,
"notice": "inspection-only",
"path": "tunnel/deploy/sysctl.conf",
"sha256": "98421fc18c49b261dd437921b91c8ad52e72d8f269a527fe38d06896132f496a"
},
{
"bytes": 16154,
"lines": 171,
"notice": "inspection-only",
"path": "tunnel/doc.go",
"sha256": "f86be2076e5d33bfb5e33c67ec7fb282db0e24c80baf0672982575313871d6a0"
},
{
"bytes": 2748,
"lines": 64,
"notice": "inspection-only",
"path": "tunnel/drain.go",
"sha256": "752e997aee5134fe72389af37f16a0c82f54717a0d6d3b56a3a20e07ce3c2673"
},
{
"bytes": 14054,
"lines": 424,
"notice": "inspection-only",
"path": "tunnel/drain_test.go",
"sha256": "8c60403b772237195c8c4f04a3ef0be5e9904b432891ee3a562581318b07e424"
},
{
"bytes": 9951,
"lines": 324,
"notice": "inspection-only",
"path": "tunnel/fakecp_test.go",
"sha256": "d5969f98b7b57681bbec9e57e34ddb683215d5f50a2279f91ea6ecceb91195ba"
},
{
"bytes": 14312,
"lines": 427,
"notice": "inspection-only",
"path": "tunnel/feed.go",
"sha256": "30d0e3881aa3507f2177c4227fc036c8c4d3e35fbdf4ef1e973f9666aace63af"
},
{
"bytes": 34338,
"lines": 962,
"notice": "inspection-only",
"path": "tunnel/feed_test.go",
"sha256": "806cdb6bd55ab1b38da418b9762e2d9817688b63f608b1e00b194fdedffd56c7"
},
{
"bytes": 3456,
"lines": 112,
"notice": "inspection-only",
"path": "tunnel/gated.go",
"sha256": "7758ac3d8409867c8efbc0c0ecb3d204454d56ec9c80f397b7f1e12620c6e52e"
},
{
"bytes": 9041,
"lines": 269,
"notice": "inspection-only",
"path": "tunnel/gated_test.go",
"sha256": "dd97e773d11ae815c055f1c85e9e38336bfd29e2eee74f8fab88553931ac3840"
},
{
"bytes": 31,
"lines": 3,
"notice": "inspection-only",
"path": "tunnel/go.mod",
"sha256": "7f6e054e5f9836856af6bac2e0bfea14ea6d1545e0f6a7045c4db8fe9ef7dc35"
},
{
"bytes": 5226,
"lines": 151,
"notice": "inspection-only",
"path": "tunnel/hardening_helpers_test.go",
"sha256": "d0b55a4840340636fbe420ecb3b063373139d5a734a39e63b881cd4ab6aa0fe2"
},
{
"bytes": 25887,
"lines": 843,
"notice": "inspection-only",
"path": "tunnel/helpers_test.go",
"sha256": "9212c296e9f2150b2188f3ed727c89213728da210c5fcee6fdff7d2a3c7395f1"
},
{
"bytes": 3278,
"lines": 120,
"notice": "inspection-only",
"path": "tunnel/hints.go",
"sha256": "f5179c76616e54b561b788b7e63fca769f3d7f6b8ddba8029adc429eb8593718"
},
{
"bytes": 2674,
"lines": 93,
"notice": "inspection-only",
"path": "tunnel/hints_test.go",
"sha256": "a3aabc10caa5401d1eccd7109354c8ad2822d2349e3dd37797549a36c424a6f9"
},
{
"bytes": 4266,
"lines": 187,
"notice": "inspection-only",
"path": "tunnel/jsonparse.go",
"sha256": "10d9a104e6eeda517aad020ee28809430a1902065fa0333cb2aa54027fdf1107"
},
{
"bytes": 11015,
"lines": 335,
"notice": "inspection-only",
"path": "tunnel/jwks.go",
"sha256": "dbfd98d8ca6342a06a26b4a32271ca4c3a5ddb16f0cf261247f92bb304c2b5a5"
},
{
"bytes": 17053,
"lines": 484,
"notice": "inspection-only",
"path": "tunnel/jwks_test.go",
"sha256": "2388e1cc03f5d5924f96cfb84da3046a9b3965a5cf6da8a2114d448d6a9aa03d"
},
{
"bytes": 12296,
"lines": 338,
"notice": "inspection-only",
"path": "tunnel/limits.go",
"sha256": "dfbaab47b90df42a33b031b144613da77dd2f4a2f488ab9e13b02825af3e4214"
},
{
"bytes": 36738,
"lines": 991,
"notice": "inspection-only",
"path": "tunnel/limits_test.go",
"sha256": "56e06711db079cd1ef2653de31170cabbc149b6f217b0d542f57a7574fd8baee"
},
{
"bytes": 6025,
"lines": 151,
"notice": "inspection-only",
"path": "tunnel/loghygiene_test.go",
"sha256": "8b80facaa30e6e6e66649aca92ff4f54989872817479fd88d805914de1889394"
},
{
"bytes": 9781,
"lines": 213,
"notice": "inspection-only",
"path": "tunnel/main.go",
"sha256": "8a0be6c0d970cba7de4dbd9e95d1d9ae48ce21f1f41fa401a28b62e046fdb42b"
},
{
"bytes": 13895,
"lines": 294,
"notice": "inspection-only",
"path": "tunnel/metrics.go",
"sha256": "ee0660cd3b88aa19237eae313441b5f8c771eaa9c8f11c4aa1741b9176375273"
},
{
"bytes": 9061,
"lines": 248,
"notice": "inspection-only",
"path": "tunnel/metrics_test.go",
"sha256": "827f6856c5692a88d6698d72ce28fcfe4583aecaa7d9ead562d8cc9062c69586"
},
{
"bytes": 5601,
"lines": 144,
"notice": "inspection-only",
"path": "tunnel/nftables_test.go",
"sha256": "01d33feb6f88072f915d1b350ed9457df2c051c26dceb7bc3ead6a230f4d4e6c"
},
{
"bytes": 2385,
"lines": 74,
"notice": "inspection-only",
"path": "tunnel/nodeproc_test.go",
"sha256": "bd578137294317aae70136121a8462799817f56d873117d0cc06a4694aa26e61"
},
{
"bytes": 9806,
"lines": 317,
"notice": "inspection-only",
"path": "tunnel/observer.go",
"sha256": "846b739ad636c61ad10d64caafda569b1620a7563602406e474771e55781acaf"
},
{
"bytes": 14046,
"lines": 379,
"notice": "inspection-only",
"path": "tunnel/observer_test.go",
"sha256": "2e9b6509848da15bcda54686bd93ee43d21a970e00fabb305b95188fa5ce7330"
},
{
"bytes": 10216,
"lines": 336,
"notice": "inspection-only",
"path": "tunnel/openai_prefixes.go",
"sha256": "12809b4af8aa3e1c69b657a8be18553af43e3857a44e31d5e18f71704b66ce4f"
},
{
"bytes": 12265,
"lines": 323,
"notice": "inspection-only",
"path": "tunnel/openai_prefixes_test.go",
"sha256": "2bfece3d472d348feede9cfbb3d75eaa3c8d6cedb1c7d676f476a771326e41a3"
},
{
"bytes": 15299,
"lines": 358,
"notice": "inspection-only",
"path": "tunnel/outage_test.go",
"sha256": "935a950e3a5eb8ec98b5ecd582e0248cbcea241168705cf9a731137f71c76f5c"
},
{
"bytes": 6679,
"lines": 159,
"notice": "inspection-only",
"path": "tunnel/policy.go",
"sha256": "e2367176c56e84c77cf622f21b79b1cb771c5af2c615e10f0cd0cd1bfd1cbea4"
},
{
"bytes": 6120,
"lines": 229,
"notice": "inspection-only",
"path": "tunnel/policy.json",
"sha256": "f736ade7541fe19c4348c777edfd17a79a99992e3ee537013a9024e9b112e461"
},
{
"bytes": 13502,
"lines": 426,
"notice": "inspection-only",
"path": "tunnel/proto.go",
"sha256": "a5474c82760c8614c4d8f2d733bb4df7c3701c5c754dfe659333d5679d05ae95"
},
{
"bytes": 13842,
"lines": 343,
"notice": "inspection-only",
"path": "tunnel/protocol_test.go",
"sha256": "59a7b338bacbc5cb415cb786c48d5ac1c1374c6670b0b92806002865d064efef"
},
{
"bytes": 2943,
"lines": 98,
"notice": "inspection-only",
"path": "tunnel/pump_buffers_test.go",
"sha256": "ca8736dfcfd495ed7f4d8b82c9b978735ea829a31bd6e73505e8b093083dd27e"
},
{
"bytes": 11760,
"lines": 311,
"notice": "inspection-only",
"path": "tunnel/selfcheck.go",
"sha256": "fe172f347bc5de5cf23394984fba5d0d630c96ee8bb2ff44376add6aaa68c126"
},
{
"bytes": 19872,
"lines": 452,
"notice": "inspection-only",
"path": "tunnel/selfcheck_test.go",
"sha256": "e565d1b1a3a13656ce1b84e4fae18f66c495019780bc7ce9c69a078c5c2a6231"
},
{
"bytes": 67625,
"lines": 1894,
"notice": "inspection-only",
"path": "tunnel/server.go",
"sha256": "1ad8c7feb651d90f1914641e55751deae687bd5dae5e5ba232ac6c8282f2c674"
},
{
"bytes": 57033,
"lines": 1450,
"notice": "inspection-only",
"path": "tunnel/server_test.go",
"sha256": "80fb7d46e02002e1a00d6cf82c958517b42f8779da11b37e2a2af46bc2be37e2"
},
{
"bytes": 7529,
"lines": 278,
"notice": "inspection-only",
"path": "tunnel/sni.go",
"sha256": "5de67027f4530cabccdf586fc1034a9db03cbe2e19e074be2d5cdd24fabaea7a"
},
{
"bytes": 15759,
"lines": 412,
"notice": "inspection-only",
"path": "tunnel/sni_test.go",
"sha256": "8881562f3b74f552c82ea3ce7f8b5f3127a805ba2e288223c1add619c393455c"
},
{
"bytes": 17475,
"lines": 577,
"notice": "inspection-only",
"path": "tunnel/splice_test.go",
"sha256": "9e9c1d6472cf16ffb20530fe43ac426cbfcbadf12a273caa3d48a375ed6726a0"
},
{
"bytes": 2885,
"lines": 62,
"notice": "inspection-only",
"path": "tunnel/stale_keys_grace_test.go",
"sha256": "2f26fd478d5abf416f6ee3d8fa5e062de4dc6849a79ba6730c3d0054a1770586"
},
{
"bytes": 3914,
"lines": 98,
"notice": "inspection-only",
"path": "tunnel/statement_test.go",
"sha256": "68d4c6d268cea8f14b9238587ab3df73c4835922388b7437626462b45b87c42c"
},
{
"bytes": 3410,
"lines": 107,
"notice": "inspection-only",
"path": "tunnel/storm_test.go",
"sha256": "d3ce5e2a8f669d5519252ba3f71d25a4b5f9764b8c9c9aad921784c09a1ede7a"
},
{
"bytes": 6102,
"lines": 153,
"notice": "inspection-only",
"path": "tunnel/unauth_reserve_test.go",
"sha256": "40171e444cb8c4b6fbe67d673b2713ea6bc871881842120289d72be126015a13"
}
],
"hash": "sha256",
"license": "inspection-only",
"notice": {
"file": "NOTICE.md",
"sha256": "2221b9b48d81c4f03f61546b9a8bc466719991f9c3c66b47fc2b74a364f8c1ba"
},
"product": "ALL",
"publisher": "Fynory LLC",
"schema": "all-encryption-source/1",
"version": "2.0.0",
"withheld": [
"tunnel/backend_loopcp.go",
"tunnel/blindmutant_test_hook.go",
"tunnel/cmd/loadgen/cp.go",
"tunnel/cmd/loadgen/device.go",
"tunnel/cmd/loadgen/flows.go",
"tunnel/cmd/loadgen/main.go",
"tunnel/cmd/loadgen/util.go",
"tunnel/tap_test_hook.go",
"tunnel/tap_test_hook_test.go",
"tunnel/testnode.go",
"tunnel/testpeer.go",
"tunnel/testpeer_peek_unix.go",
"tunnel/testpeer_peek_windows.go"
]
}
NOTICE.md 25 lines · 1.5 KB
# ALL: código del cifrado publicado para inspección (versión 2.0.0)
© 2026 Fynory LLC. Todos los derechos reservados.
Este código se publica solo para que puedas **leerlo y verificarlo**: que los archivos del manifiesto que corren en tu computadora sean los publicados,
y que el cifrado haga lo que dice. La comprobación solo mira los archivos listados en MANIFEST.json; no revisa el resto de ALL.
No puedes copiarlo, usarlo, modificarlo, compilarlo, redistribuirlo, publicarlo en otro sitio ni crear obras derivadas a partir de él, ni siquiera en parte.
Ninguna licencia se concede, salvo la de leer y verificar. Cualquier otro uso requiere permiso escrito de Fynory LLC.
Los archivos son bytes idénticos a los del programa instalado. Su SHA-256 está en MANIFEST.json.
---
# ALL: encryption code published for inspection (version 2.0.0)
© 2026 Fynory LLC. All rights reserved.
This code is published only so that you can **read it and verify it**: that the manifest files that run on your computer are the published ones, and that
the encryption does what it says. The check looks only at the files listed in MANIFEST.json; it does not review the rest of ALL.
You may not copy, use, modify, compile, redistribute, post elsewhere, or create derivative works from it, not even in part. No license is granted other than
the right to read and verify. Any other use requires the written permission of Fynory LLC.
The files are byte-identical to the files of the installed program. Their SHA-256 is in MANIFEST.json.
files/contracts/e2e_contracts.py 855 lines · 40.9 KB
"""ALL 2.0 contracts, Python reference verifier (the contract).
The spec is `contracts/policy.json` (every constant), `contracts/schemas/` (shapes, line grammar) and `contracts/vectors/` (golden inputs and the
verdict every verifier must give). This module is the Python implementation of the checks the vectors exercise; the Go one is
`tunnel/contracts_test.go` and the TypeScript one is `contracts/ts/verify.ts`. All three give the same verdict AND the same error word for
every vector (the contract test suite, tunnel/contracts_test.go, the Worker contract test). It is a reference for tests and
for the Python fake node (fake_node.py): production code is written by the packages that own it, and reads the numbers from policy.json.
What it checks, in this order of failure (the first failing step names the error):
* lines parse_line / format_line (the wire grammar of schemas/wire-v2.grammar.json)
* bodies Schemas.validate (the JSON Schema subset the schemas use) and check_feed (the cross-field rules of the feed)
* tokens verify_session, verify_proof (DPoP), verify_node_assertion, verify_statement, parse_jwks, parse_node_keys
* surface surface_mac / surface_proof (the HMAC identity proof of the E2E listener)
* names label_ok / fqdn_ok
* nodes check_transcript (does a recorded node behave as the contract demands: renew re-authorizes, host from the control plane only...)
Pure functions: the clock, the replay set and the key sets are arguments, never globals.
"""
from __future__ import annotations
import base64
import hashlib
import hmac
import ipaddress
import json
import re
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from cryptography.exceptions import InvalidSignature
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
ROOT = Path(__file__).resolve().parent
POLICY_PATH = ROOT / "policy.json"
SCHEMA_DIR = ROOT / "schemas"
VECTOR_DIR = ROOT / "vectors"
def load_policy(path: Path | None = None) -> dict[str, Any]:
return json.loads((path or POLICY_PATH).read_text(encoding="utf-8"))
POLICY = load_policy()
def pol(dotted: str, policy: dict[str, Any] | None = None) -> Any:
"""`pol("protocol.line_max_bytes")`: the one way this module reads a number."""
value: Any = POLICY if policy is None else policy
for part in dotted.split("."):
value = value[part]
return value
# ------------------------------------------------------------------------------------------------------------------------ regular expressions
def compile_anchored(pattern: str) -> re.Pattern[str]:
"""Every pattern of the contracts is written ^...$ and means "the whole string": Python's `$` also matches before a final newline, so the
anchors are dropped and `fullmatch` is used (JavaScript and Go anchor strictly by themselves)."""
if not (pattern.startswith("^") and pattern.endswith("$")):
raise ValueError("contract patterns are anchored: " + pattern)
return re.compile(pattern[1:-1])
LABEL_RE = compile_anchored(pol("identity.label_regex"))
FQDN_RE = compile_anchored(pol("identity.fqdn_regex"))
DEVICE_ID_RE = compile_anchored(pol("identity.device_id_regex"))
STREAM_ID_RE = compile_anchored(pol("identity.stream_id_regex"))
ATTACH_SECRET_RE = compile_anchored(pol("identity.attach_secret_regex"))
NONCE_RE = compile_anchored(pol("identity.nonce_regex"))
JTI_RE = compile_anchored(pol("identity.jti_regex"))
KID_RE = compile_anchored(pol("identity.kid_regex"))
def label_ok(value: Any) -> bool:
return isinstance(value, str) and LABEL_RE.fullmatch(value) is not None
def host_pattern(zone: str) -> re.Pattern[str]:
"""The detector a device (or a node) builds for ITS OWN address in a zone: the policy's label pattern, one dot, the zone. The label part is
never typed again anywhere (the lint refuses a copy of it). Anchored with \\A and \\Z (not `$`, which also matches before a final newline), so
match, search and fullmatch all give the verdict the Go and TypeScript detectors give."""
return re.compile(r"\A" + pol("identity.label_regex")[1:-1] + r"\." + re.escape(zone) + r"\Z")
def fqdn_ok(value: Any, zones: list[str] | None = None) -> bool:
"""A device host: <label>.<zone>. With `zones`, the part after the label must be one of them (several zones are allowed by the contract)."""
if not isinstance(value, str) or len(value) > pol("identity.fqdn_max_length") or FQDN_RE.fullmatch(value) is None:
return False
return zones is None or value.split(".", 1)[1] in zones
# ------------------------------------------------------------------------------------------------------------------------ base64url, JSON
def b64u_encode(raw: bytes) -> str:
return base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii")
_B64U = re.compile(r"[A-Za-z0-9_-]+")
def b64u_decode(text: str) -> bytes:
"""Strict: the URL alphabet only, no padding, and the canonical encoding (re-encoding gives the same text)."""
if not isinstance(text, str) or not _B64U.fullmatch(text):
raise ValueError("base64url")
raw = base64.urlsafe_b64decode(text + "=" * (-len(text) % 4))
if b64u_encode(raw) != text:
raise ValueError("base64url")
return raw
def sha256_b64u(data: bytes | str) -> str:
return b64u_encode(hashlib.sha256(data.encode("utf-8") if isinstance(data, str) else data).digest())
def canonical_json(value: Any) -> str:
"""The form of a JSON body on a wire line: keys sorted, no whitespace, ASCII only. Bodies hold integers, never floats."""
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
def _no_floats(value: Any) -> Any:
if isinstance(value, float):
if value.is_integer():
return int(value)
raise ValueError("float")
if isinstance(value, dict):
return {k: _no_floats(v) for k, v in value.items()}
if isinstance(value, list):
return [_no_floats(v) for v in value]
return value
def loads(text: str) -> Any:
"""json.loads that refuses duplicate keys, NaN and Infinity (a body with a repeated key is ambiguous between implementations)."""
def pairs(items: list[tuple[str, Any]]) -> dict[str, Any]:
out: dict[str, Any] = {}
for key, item in items:
if key in out:
raise ValueError("duplicate key")
out[key] = item
return out
def constant(_: str) -> Any:
raise ValueError("constant")
return json.loads(text, object_pairs_hook=pairs, parse_constant=constant)
# ------------------------------------------------------------------------------------------------------------------------ JSON Schema subset
def valid_ip(value: Any) -> bool:
"""`format: ip`: a textual IPv4 or IPv6 address, lower case, no zone, no brackets, no spaces."""
if not isinstance(value, str) or not value or value != value.lower() or any(c in value for c in "%[] \t\r\n/"):
return False
try:
ipaddress.ip_address(value)
except ValueError:
return False
return True
def _type_ok(kind: str, value: Any) -> bool:
if kind == "string":
return isinstance(value, str)
if kind == "boolean":
return isinstance(value, bool)
if kind == "null":
return value is None
if kind == "object":
return isinstance(value, dict)
if kind == "array":
return isinstance(value, list)
if kind == "integer":
return (isinstance(value, int) and not isinstance(value, bool)) or (isinstance(value, float) and value.is_integer())
if kind == "number":
return isinstance(value, (int, float)) and not isinstance(value, bool)
return False
def _equal(a: Any, b: Any) -> bool:
"""JSON equality: 1 is not true, 1 == 1.0, objects by content."""
if isinstance(a, bool) or isinstance(b, bool):
return isinstance(a, bool) and isinstance(b, bool) and a == b
if isinstance(a, (int, float)) and isinstance(b, (int, float)):
return a == b
if isinstance(a, dict) and isinstance(b, dict):
return a.keys() == b.keys() and all(_equal(a[k], b[k]) for k in a)
if isinstance(a, list) and isinstance(b, list):
return len(a) == len(b) and all(_equal(x, y) for x, y in zip(a, b))
return type(a) is type(b) and a == b
class Schemas:
"""The schemas of contracts/schemas/ by file name, and a validator for the keywords they use: $ref, type, const, enum, pattern, format
(ip), minLength, maxLength, minimum, maximum, required, properties, additionalProperties (false), items, minItems, maxItems, uniqueItems,
oneOf. Anything else in a schema is ignored (titles, $comment, $id), so a keyword added later must be added to all three validators."""
def __init__(self, directory: Path | None = None):
self.docs: dict[str, Any] = {}
for path in sorted((directory or SCHEMA_DIR).glob("*.schema.json")):
self.docs[path.name] = json.loads(path.read_text(encoding="utf-8"))
self._patterns: dict[str, re.Pattern[str]] = {}
def validate(self, name: str, instance: Any) -> list[str]:
errors: list[str] = []
self._check(name, self.docs[name], instance, "", errors)
return errors
def _resolve(self, doc: str, ref: str) -> tuple[str, Any]:
file, _, pointer = ref.partition("#")
target = file or doc
node: Any = self.docs[target]
if pointer:
for part in pointer.lstrip("/").split("/"):
node = node[part.replace("~1", "/").replace("~0", "~")]
return target, node
def _pattern(self, text: str) -> re.Pattern[str]:
if text not in self._patterns:
self._patterns[text] = compile_anchored(text)
return self._patterns[text]
def _check(self, doc: str, schema: Any, value: Any, path: str, errors: list[str]) -> None:
if "$ref" in schema:
target, node = self._resolve(doc, schema["$ref"])
self._check(target, node, value, path, errors)
if "const" in schema and not _equal(value, schema["const"]):
errors.append(f"{path or '/'}: const")
if "enum" in schema and not any(_equal(value, item) for item in schema["enum"]):
errors.append(f"{path or '/'}: enum")
if "type" in schema:
kinds = schema["type"] if isinstance(schema["type"], list) else [schema["type"]]
if not any(_type_ok(kind, value) for kind in kinds):
errors.append(f"{path or '/'}: type")
return
if "oneOf" in schema:
hits = 0
for option in schema["oneOf"]:
trial: list[str] = []
self._check(doc, option, value, path, trial)
hits += not trial
if hits != 1:
errors.append(f"{path or '/'}: oneOf")
if isinstance(value, str):
if "minLength" in schema and len(value) < schema["minLength"]:
errors.append(f"{path or '/'}: minLength")
if "maxLength" in schema and len(value) > schema["maxLength"]:
errors.append(f"{path or '/'}: maxLength")
if "pattern" in schema and self._pattern(schema["pattern"]).fullmatch(value) is None:
errors.append(f"{path or '/'}: pattern")
if schema.get("format") == "ip" and not valid_ip(value):
errors.append(f"{path or '/'}: format")
if isinstance(value, (int, float)) and not isinstance(value, bool):
if "minimum" in schema and value < schema["minimum"]:
errors.append(f"{path or '/'}: minimum")
if "maximum" in schema and value > schema["maximum"]:
errors.append(f"{path or '/'}: maximum")
if isinstance(value, dict):
for key in schema.get("required", []):
if key not in value:
errors.append(f"{path}/{key}: required")
props = schema.get("properties", {})
for key, item in value.items():
if key in props:
self._check(doc, props[key], item, f"{path}/{key}", errors)
elif schema.get("additionalProperties") is False:
errors.append(f"{path}/{key}: additionalProperties")
if isinstance(value, list):
if "minItems" in schema and len(value) < schema["minItems"]:
errors.append(f"{path or '/'}: minItems")
if "maxItems" in schema and len(value) > schema["maxItems"]:
errors.append(f"{path or '/'}: maxItems")
if schema.get("uniqueItems") and any(_equal(a, b) for i, a in enumerate(value) for b in value[i + 1:]):
errors.append(f"{path or '/'}: uniqueItems")
if "items" in schema:
for index, item in enumerate(value):
self._check(doc, schema["items"], item, f"{path}/{index}", errors)
# ------------------------------------------------------------------------------------------------------------------------ the wire grammar
class ProtocolError(Exception):
"""A line that is not valid. `code` is one of protocol.error_codes: the word the node answers with (ERR <code>)."""
def __init__(self, code: str):
super().__init__(code)
self.code = code
GRAMMAR = json.loads((SCHEMA_DIR / "wire-v2.grammar.json").read_text(encoding="utf-8"))
SCHEMAS = Schemas()
def _field_ok(kind: str, token: str) -> bool:
spec = GRAMMAR["types"][kind]
if "max_bytes_from" in spec and len(token) > pol(spec["max_bytes_from"]):
return False
if "pattern" in spec and compile_anchored(spec["pattern"]).fullmatch(token) is None:
return False
if spec.get("format") == "ip" and not valid_ip(token):
return False
if "enum_from" in spec and token not in pol(spec["enum_from"]):
return False
return True
def parse_line(line: str, direction: str) -> dict[str, Any]:
"""Parse one protocol line (the text before the LF) travelling in `direction` ("d2n" or "n2d"). Returns {"verb": ..., <field>: ...}; HINT
gives "addresses", OK gives "body". Raises ProtocolError."""
if not isinstance(line, str) or any(not 0x20 <= ord(c) <= 0x7E for c in line):
raise ProtocolError("bad_command")
verb, space, rest = line.partition(" ")
spec = GRAMMAR["verbs"].get(verb)
cap = pol(spec["max"]) if spec and "max" in spec else pol("protocol.line_max_bytes")
if len(line) + 1 > cap:
raise ProtocolError("line_too_long")
if spec is None or spec["dir"] not in (direction, "both") or line == "" or (space and rest == ""):
raise ProtocolError("bad_command")
out: dict[str, Any] = {"verb": verb}
if "json" in spec:
if not space:
raise ProtocolError("bad_command")
try:
body = _no_floats(loads(rest))
except ValueError:
raise ProtocolError("bad_command") from None
if SCHEMAS.validate(spec["json"], body):
raise ProtocolError("bad_command")
out["body"] = body
return out
tokens = rest.split(" ") if space else []
if any(token == "" for token in tokens):
raise ProtocolError("bad_command")
if "repeat" in spec:
rule = spec["repeat"]
if not rule["min"] <= len(tokens) <= pol(rule["max_from"]) or not all(_field_ok(rule["type"], t) for t in tokens):
raise ProtocolError("bad_command")
out["addresses"] = tokens
return out
fields = spec.get("fields", [])
optional = spec.get("optional", [])
if not len(fields) <= len(tokens) <= len(fields) + len(optional):
raise ProtocolError("bad_command")
for rule, token in zip(fields + optional, tokens):
if "equals" in rule and token != rule["equals"]:
raise ProtocolError(rule["error_if_number"] if _field_ok(rule["type"], token) else "bad_command")
if not _field_ok(rule["type"], token):
raise ProtocolError(rule.get("error", "bad_command"))
out[rule["name"]] = token
return out
def format_line(verb: str, **fields: Any) -> str:
"""The canonical line for a message (the inverse of parse_line, which it is checked against): the field values in grammar order; OK takes
`body`, HINT takes `addresses`. Raises ProtocolError when the result would not parse."""
spec = GRAMMAR["verbs"][verb]
if "json" in spec:
line = f"{verb} {canonical_json(fields['body'])}"
elif "repeat" in spec:
line = " ".join([verb, *fields["addresses"]])
else:
names = [rule["name"] for rule in spec.get("fields", []) + spec.get("optional", []) if rule["name"] in fields]
line = " ".join([verb, *(str(fields[name]) for name in names)])
direction = "d2n" if spec["dir"] == "both" else spec["dir"]
parse_line(line, direction)
return line
# ------------------------------------------------------------------------------------------------------------------------ tokens
@dataclass
class Result:
ok: bool
error: str | None = None
claims: dict[str, Any] = field(default_factory=dict)
jti: str | None = None
@staticmethod
def fail(code: str) -> "Result":
return Result(False, code)
class Reject(Exception):
def __init__(self, code: str):
super().__init__(code)
self.code = code
def _jws_parts(token: Any, max_bytes: int) -> tuple[dict[str, Any], dict[str, Any], bytes, bytes]:
try:
if not isinstance(token, str) or not token.isascii() or len(token) > max_bytes:
raise ValueError()
parts = token.split(".")
if len(parts) != 3:
raise ValueError()
header = loads(b64u_decode(parts[0]).decode("utf-8"))
claims = loads(b64u_decode(parts[1]).decode("utf-8"))
signature = b64u_decode(parts[2]) if parts[2] else b""
if not isinstance(header, dict) or not isinstance(claims, dict):
raise ValueError()
if any(name in header for name in ("crit", "b64", "jku", "x5u", "x5c")):
raise ValueError()
claims = _no_floats(claims)
except (ValueError, UnicodeError):
raise Reject("malformed_token") from None
return header, claims, signature, f"{parts[0]}.{parts[1]}".encode("ascii")
def _ed25519_ok(x: str, signature: bytes, signing_input: bytes) -> bool:
"""Ed25519 verification; a signature that is not 64 bytes is simply not valid."""
if len(signature) != 64:
return False
try:
Ed25519PublicKey.from_public_bytes(b64u_decode(x)).verify(signature, signing_input)
return True
except (InvalidSignature, ValueError):
return False
def _is_int(value: Any) -> bool:
return isinstance(value, int) and not isinstance(value, bool)
def _jwk_x(value: Any) -> str:
"""The `x` of a public Ed25519 JWK (kty OKP, crv Ed25519, no private part), or a Reject."""
if (not isinstance(value, dict) or value.get("kty") != "OKP" or value.get("crv") != "Ed25519" or "d" in value
or not isinstance(value.get("x"), str) or len(value["x"]) != 43):
raise Reject("malformed_token")
try:
if len(b64u_decode(value["x"])) != 32:
raise ValueError()
except ValueError:
raise Reject("malformed_token") from None
return value["x"]
def parse_jwks(jwks: Any) -> dict[str, str]:
"""kid -> x of the control plane's keys (schemas/jwks.schema.json, at most policy.jwks.max_keys, no repeated kid), or Reject("bad_jwks")."""
if SCHEMAS.validate("jwks.schema.json", jwks):
raise Reject("bad_jwks")
keys: dict[str, str] = {}
for key in jwks["keys"]:
if key["kid"] in keys:
raise Reject("bad_jwks")
keys[key["kid"]] = key["x"]
return keys
def parse_node_keys(doc: Any) -> dict[str, tuple[str, str]]:
"""kid -> (node id, x) of TUNNEL_NODE_KEYS, or Reject("bad_node_keys")."""
if SCHEMAS.validate("node-keys.schema.json", doc):
raise Reject("bad_node_keys")
keys: dict[str, tuple[str, str]] = {}
for key in doc["keys"]:
if key["kid"] in keys:
raise Reject("bad_node_keys")
keys[key["kid"]] = (key["node"], key["x"])
return keys
def _signed_by_jwks(token: Any, keys: dict[str, str], typ: str) -> tuple[dict[str, Any], dict[str, Any]]:
header, claims, signature, signing_input = _jws_parts(token, pol("protocol.statement_line_max_bytes"))
if header.get("alg") != "EdDSA":
raise Reject("bad_alg")
kid = header.get("kid")
if not isinstance(kid, str) or KID_RE.fullmatch(kid) is None or kid not in keys:
raise Reject("unknown_kid")
if not _ed25519_ok(keys[kid], signature, signing_input):
raise Reject("bad_signature")
if header.get("typ") != typ or claims.get("typ") != typ:
raise Reject("wrong_type")
return header, claims
def _claim_ok(claims: dict[str, Any], schema: str, ignore: tuple[str, ...] = ()) -> bool:
"""Shape of the claims by their schema, without the keywords checked earlier under their own error word."""
errors = SCHEMAS.validate(schema, claims)
return not [e for e in errors if not any(e.startswith(f"/{name}:") for name in ignore)]
def verify_session(token: Any, *, now: int, audience: str, keys: dict[str, str], skew: int) -> Result:
"""A device session (typ device-session, ver 2) as the node must accept it at HELLO and RENEW. Order of failure: malformed_token, bad_alg,
unknown_kid, bad_signature, wrong_type, wrong_issuer, wrong_audience, bad_claims, bad_lifetime, not_yet_valid, expired."""
try:
if not isinstance(token, str) or len(token) > pol("protocol.jwt_max_bytes"):
raise Reject("malformed_token")
_, claims = _signed_by_jwks(token, keys, "device-session")
if claims.get("ver") != 2 or isinstance(claims.get("ver"), bool):
raise Reject("wrong_type")
if claims.get("iss") != pol("identity.issuer"):
raise Reject("wrong_issuer")
if claims.get("aud") != audience:
raise Reject("wrong_audience")
if not _claim_ok(claims, "session-claims.schema.json", ("aud",)):
raise Reject("bad_claims")
if claims["exp"] - claims["iat"] != pol("authorization.session_ttl_s"):
raise Reject("bad_lifetime")
if claims["iat"] > now + skew:
raise Reject("not_yet_valid")
if claims["exp"] <= now:
raise Reject("expired")
return Result(True, claims=claims, jti=claims["jti"])
except Reject as reject:
return Result.fail(reject.code)
def verify_proof(proof: Any, *, now: int, session_token: str, session_claims: dict[str, Any], nonce: str, htu: str, skew: int,
seen: set[str] | frozenset[str]) -> Result:
"""The DPoP proof of PROOF. Order of failure: malformed_token, bad_alg, wrong_type, wrong_key, bad_signature, wrong_subject,
wrong_audience, wrong_htm, wrong_htu, wrong_ath, wrong_nonce, bad_claims, not_yet_valid, stale_proof, replayed_jti."""
try:
if not isinstance(proof, str) or len(proof) > pol("protocol.jwt_max_bytes"):
raise Reject("malformed_token")
header, claims, signature, signing_input = _jws_parts(proof, pol("protocol.jwt_max_bytes"))
if header.get("alg") != "EdDSA":
raise Reject("bad_alg")
if header.get("typ") != "dpop+jwt":
raise Reject("wrong_type")
x = _jwk_x(header.get("jwk"))
if x != session_claims["cnf"]["jwk"]["x"]:
raise Reject("wrong_key")
if not _ed25519_ok(x, signature, signing_input):
raise Reject("bad_signature")
if claims.get("sub") != session_claims["sub"]:
raise Reject("wrong_subject")
if claims.get("aud") != pol("identity.aud_tunnel"):
raise Reject("wrong_audience")
if claims.get("htm") != pol("protocol.proof_htm"):
raise Reject("wrong_htm")
if claims.get("htu") != htu:
raise Reject("wrong_htu")
if claims.get("ath") != sha256_b64u(session_token):
raise Reject("wrong_ath")
if claims.get("nonce") != nonce:
raise Reject("wrong_nonce")
if not _claim_ok(claims, "proof-claims.schema.json", ("htu",)):
raise Reject("bad_claims")
if claims["iat"] > now + skew:
raise Reject("not_yet_valid")
if claims["iat"] + pol("authorization.proof_ttl_s") <= now:
raise Reject("stale_proof")
if claims["jti"] in seen:
raise Reject("replayed_jti")
return Result(True, claims=claims, jti=claims["jti"])
except Reject as reject:
return Result.fail(reject.code)
def verify_node_assertion(token: Any, *, now: int, keys: dict[str, tuple[str, str]], method: str, url: str, body: bytes, skew: int,
seen: set[str] | frozenset[str]) -> Result:
"""A node's request authentication (header x-all-node-assertion). Order of failure: malformed_token, bad_alg, unknown_kid, bad_signature,
wrong_type, wrong_audience, wrong_subject, wrong_htm, wrong_htu, bad_claims, bad_lifetime, not_yet_valid, expired, wrong_body_hash,
replayed_jti."""
try:
header, claims, signature, signing_input = _jws_parts(token, pol("protocol.jwt_max_bytes"))
if header.get("alg") != "EdDSA":
raise Reject("bad_alg")
kid = header.get("kid")
if not isinstance(kid, str) or KID_RE.fullmatch(kid) is None or kid not in keys:
raise Reject("unknown_kid")
node, x = keys[kid]
if not _ed25519_ok(x, signature, signing_input):
raise Reject("bad_signature")
if header.get("typ") != "node-assertion" or claims.get("typ") != "node-assertion" or claims.get("ver") != 1:
raise Reject("wrong_type")
if claims.get("aud") != pol("identity.aud_node_assertion"):
raise Reject("wrong_audience")
if claims.get("sub") != node:
raise Reject("wrong_subject")
if claims.get("htm") != method:
raise Reject("wrong_htm")
if claims.get("htu") != url:
raise Reject("wrong_htu")
if not _claim_ok(claims, "node-assertion-claims.schema.json", ("htm", "htu", "aud", "sub")):
raise Reject("bad_claims")
if claims["exp"] - claims["iat"] != pol("authorization.node_assertion_ttl_s"):
raise Reject("bad_lifetime")
if claims["iat"] > now + skew:
raise Reject("not_yet_valid")
if claims["exp"] <= now:
raise Reject("expired")
if claims["bh"] != sha256_b64u(body):
raise Reject("wrong_body_hash")
if claims["jti"] in seen:
raise Reject("replayed_jti")
return Result(True, claims=claims, jti=claims["jti"])
except Reject as reject:
return Result.fail(reject.code)
def verify_statement(token: Any, *, now: int, keys: dict[str, str], device_id: str, last_seq: int) -> Result:
"""The control plane's signed access statement as the device must accept it (STATEMENT, authorize). Order of failure: malformed_token,
bad_alg, unknown_kid, bad_signature, wrong_type, wrong_issuer, wrong_audience, wrong_subject, bad_claims, bad_lifetime, not_yet_valid,
stale_seq. A statement that has merely EXPIRED still verifies (a stored paywall keeps gating): the caller compares exp with its own clock.
last_seq is the highest seq accepted so far (0 for none); the same seq again is accepted (idempotent), a lower one is stale_seq."""
try:
_, claims = _signed_by_jwks(token, keys, "device-access")
if claims.get("ver") != 1 or isinstance(claims.get("ver"), bool):
raise Reject("wrong_type")
if claims.get("iss") != pol("identity.issuer"):
raise Reject("wrong_issuer")
if claims.get("aud") != pol("identity.aud_access"):
raise Reject("wrong_audience")
if claims.get("sub") != device_id:
raise Reject("wrong_subject")
if (not _claim_ok(claims, "statement-claims.schema.json", ("aud", "sub")) or ("paywall" in claims) != (claims["mode"] == "paywall")
or ("paused" in claims and "resumed" in claims)):
raise Reject("bad_claims")
ttl = claims["exp"] - claims["iat"]
if not 0 < ttl <= pol("authorization.access_ttl_paywall_s" if claims["mode"] == "paywall" else "authorization.access_ttl_paid_s"):
raise Reject("bad_lifetime")
if claims["iat"] > now + pol("authorization.access_future_slack_s"):
raise Reject("not_yet_valid")
if claims["seq"] < last_seq:
raise Reject("stale_seq")
return Result(True, claims=claims, jti=claims["jti"])
except Reject as reject:
return Result.fail(reject.code)
# ------------------------------------------------------------------------------------------------------------------------ the feed
def check_feed(body: Any, since: int) -> list[str]:
"""Schema plus the rules a schema cannot say: events strictly increasing, all after `since`, epoch not below the last event, and the two
global events (kill, kill_clear) carry no device, every other event carries both device and label."""
errors = SCHEMAS.validate("feed-response.schema.json", body)
if errors:
return errors
previous = since
for index, event in enumerate(body["events"]):
if event["epoch"] <= previous:
errors.append(f"/events/{index}: order")
previous = max(previous, event["epoch"])
is_global = event["type"] in ("kill", "kill_clear")
if is_global != (event["device_id"] is None and event["label"] is None):
errors.append(f"/events/{index}: scope")
if (event["device_id"] is None) != (event["label"] is None):
errors.append(f"/events/{index}: scope")
if body["epoch"] < previous:
errors.append("/epoch: behind events")
return errors
# ------------------------------------------------------------------------------------------------------------------------ surface identity proof
def _surface_key(secret: str) -> bytes:
raw = b64u_decode(secret)
if len(raw) != 32:
raise ValueError("secret")
return raw
def surface_mac(secret: str, nonce: str) -> str:
"""The MAC the terminator puts on its challenge: only a holder of the per-start secret (the agent) gets an answer; anybody else gets a 404."""
return b64u_encode(hmac.new(_surface_key(secret), b"all-surface-challenge-v1\n" + nonce.encode("ascii"), hashlib.sha256).digest())
def surface_proof(secret: str, nonce: str, pid: int, port: int) -> str:
"""What the listener answers: it proves it holds the secret of the process surfaces.json names, for this nonce, pid and port."""
message = b"all-surface-proof-v1\n" + nonce.encode("ascii") + b"\n" + str(pid).encode() + b"\n" + str(port).encode()
return b64u_encode(hmac.new(_surface_key(secret), message, hashlib.sha256).digest())
def verify_surface_challenge(secret: str, nonce: str, mac: Any) -> Result:
try:
if not isinstance(nonce, str) or NONCE_RE.fullmatch(nonce) is None or not isinstance(mac, str):
return Result.fail("bad_challenge")
return Result(True) if hmac.compare_digest(surface_mac(secret, nonce), mac) else Result.fail("bad_challenge")
except ValueError:
return Result.fail("bad_challenge")
def verify_surface_proof(secret: str, nonce: str, pid: int, port: int, proof: Any) -> Result:
try:
if not isinstance(proof, str) or not isinstance(nonce, str) or NONCE_RE.fullmatch(nonce) is None:
return Result.fail("bad_proof")
return Result(True) if hmac.compare_digest(surface_proof(secret, nonce, pid, port), proof) else Result.fail("bad_proof")
except ValueError:
return Result.fail("bad_proof")
def identity_request(host: str, port: int, nonce: str, mac: str) -> str:
"""The exact request head the terminator sends on the connection it is about to hand to ChatGPT (before the first ChatGPT byte)."""
return (f"GET {pol('surface.identity_path')} HTTP/1.1\r\nHost: {host}:{port}\r\n{pol('surface.identity_challenge_header')}: {nonce}\r\n"
f"{pol('surface.identity_mac_header')}: {mac}\r\nConnection: keep-alive\r\n\r\n")
# ------------------------------------------------------------------------------------------------------------------------ node behaviour
@dataclass
class _Conn:
phase: str = "new" # new, hello, ready, closed
session: str | None = None
claims: dict[str, Any] | None = None
session_ok: bool = False
kind: str = "hello"
nonce: str | None = None
proof: str = "none" # none, ok, replayed, invalid
authorized: bool = False
reply: dict[str, Any] | None = None
host: str | None = None
status: str | None = None
valid_until: int = 0
superseded: bool = False
sub: str | None = None
def _violation(code: str) -> Result:
return Result(False, code)
def check_transcript(doc: dict[str, Any]) -> Result:
"""Does the recorded behaviour of ONE node obey the contract? `doc` is {"ctx": {...}, "events": [...]} .
Returns ok, or the first violation word. The rules, each with its word:
handshake challenge_for_invalid_session, ok_without_proof, ok_with_invalid_proof, replayed_proof_accepted, ok_without_authorize,
renew_without_authorize, bad_authorize_request, host_not_from_control_plane, ok_mismatch_control_plane, bad_line
streams open_before_ok, open_when_paused, open_after_valid_until, open_without_client_hello, open_wrong_host, open_wrong_client,
duplicate_stream
attach attach_unknown_accepted, attach_wrong_secret_accepted, attach_secret_reused, attach_after_timeout, attach_refused_good_secret
lifetime no_close_at_valid_until, replaced_not_closed (both are judged at the `end` event: a recording without one is transcript_without_end)
"""
ctx = doc["ctx"]
keys = parse_jwks(ctx["jwks"])
skew = ctx["skew"]
conns: dict[str, _Conn] = {}
seen: set[str] = set()
streams: dict[str, dict[str, Any]] = {}
secrets_used: set[str] = set()
hellos: dict[str, list[tuple[str, str]]] = {} # per host: the (peer_ip, class) of each complete ClientHello still waiting for its OPEN
ended = False
now = 0
for event in doc["events"]:
now = event["t"]
kind = event["ev"]
conn = conns.setdefault(event["conn"], _Conn()) if "conn" in event and kind in ("recv", "send", "cp", "cp_reply") else None
if kind == "client_hello":
if event["complete"]:
hellos.setdefault(event["sni"], []).append((event["peer_ip"], event["class"]))
elif kind == "recv":
assert conn is not None
try:
message = parse_line(event["line"], "d2n")
except ProtocolError:
continue
verb = message["verb"]
if verb in ("HELLO", "RENEW") and (conn.phase == "new" if verb == "HELLO" else conn.phase == "ready"):
result = verify_session(message["session"], now=now, audience=pol("identity.aud_tunnel"), keys=keys, skew=skew)
conn.session_ok = result.ok and (verb == "HELLO" or result.claims["sub"] == conn.sub)
conn.session = message["session"]
conn.claims = result.claims if result.ok else None
conn.kind = "hello" if verb == "HELLO" else "renew"
conn.nonce = None
conn.proof, conn.authorized, conn.reply = "none", False, None
if verb == "HELLO":
conn.phase = "hello"
elif verb == "PROOF" and conn.session_ok and conn.nonce is not None:
result = verify_proof(message["proof"], now=now, session_token=conn.session, session_claims=conn.claims, nonce=conn.nonce,
htu=ctx["node_url"], skew=skew, seen=seen)
conn.proof = "ok" if result.ok else ("replayed" if result.error == "replayed_jti" else "invalid")
conn.authorized, conn.reply = False, None
if result.ok and result.jti is not None:
seen.add(result.jti)
elif kind == "cp":
assert conn is not None and event["call"] == "authorize"
body = event["body"]
if (SCHEMAS.validate("authorize-request.schema.json", body) or conn.claims is None or body["device_id"] != conn.claims["sub"]
or body["session_jti"] != conn.claims["jti"] or body["reason"] != conn.kind or body["node"] != ctx["node_id"]):
return _violation("bad_authorize_request")
if conn.proof == "ok":
conn.authorized = True
elif kind == "cp_reply":
assert conn is not None
conn.reply = event if conn.authorized else None
elif kind == "send":
assert conn is not None
try:
message = parse_line(event["line"], "n2d")
except ProtocolError:
return _violation("bad_line")
verb = message["verb"]
if verb == "CHALLENGE":
if not conn.session_ok or conn.phase not in ("hello", "ready"):
return _violation("challenge_for_invalid_session")
conn.nonce = message["nonce"]
elif verb == "OK":
if event["line"] != "OK " + canonical_json(message["body"]):
return _violation("bad_line") # the node sends the canonical form (the device's parser is lenient: see docs section 2)
if conn.proof == "none":
return _violation("ok_without_proof")
if conn.proof == "replayed":
return _violation("replayed_proof_accepted")
if conn.proof != "ok":
return _violation("ok_with_invalid_proof")
reply = conn.reply
if not conn.authorized or reply is None or reply["status"] != 200:
return _violation("renew_without_authorize" if conn.kind == "renew" else "ok_without_authorize")
body = message["body"]
answer = reply["body"]
if body["host"] != answer["host"]:
return _violation("host_not_from_control_plane")
if (body["status"] != answer["status"] or body["valid_until"] != answer["valid_until"] or body["policy"] != answer["policy"]
or body["node"] != ctx["node_id"]):
return _violation("ok_mismatch_control_plane")
conn.phase, conn.host, conn.status, conn.valid_until, conn.sub = "ready", body["host"], body["status"], body["valid_until"], conn.claims["sub"]
conn.proof, conn.authorized, conn.reply = "none", False, None
for other_name, other in conns.items():
if other is not conn and other.phase == "ready" and other.sub == conn.sub:
other.superseded = True
elif verb == "OPEN":
if conn.phase != "ready" or conn.status is None:
return _violation("open_before_ok")
if conn.status == "paused":
return _violation("open_when_paused")
if now >= conn.valid_until:
return _violation("open_after_valid_until")
pending = hellos.get(conn.host or "", [])
if not pending:
return _violation("open_wrong_host" if any(hellos.values()) else "open_without_client_hello")
if (message["peer_ip"], message["class"]) not in pending:
return _violation("open_wrong_client")
if message["stream_id"] in streams or message["secret"] in secrets_used:
return _violation("duplicate_stream")
pending.remove((message["peer_ip"], message["class"]))
streams[message["stream_id"]] = {"secret": message["secret"], "t": now, "consumed": False}
secrets_used.add(message["secret"])
elif verb == "CLOSE":
conn.phase = "closed"
elif kind == "attach":
try:
message = parse_line(event["line"], "d2n")
except ProtocolError:
continue
known = streams.get(message["stream_id"])
good = known is not None and hmac.compare_digest(known["secret"], message["secret"])
fresh = good and not known["consumed"] and now - known["t"] <= pol("protocol.attach_timeout_s")
if event["result"] == "ok":
if known is None:
return _violation("attach_unknown_accepted")
if not good:
return _violation("attach_wrong_secret_accepted")
if known["consumed"]:
return _violation("attach_secret_reused")
if not fresh:
return _violation("attach_after_timeout")
known["consumed"] = True
elif fresh:
return _violation("attach_refused_good_secret")
elif kind == "end":
ended = True
for other in conns.values():
if other.phase == "ready" and other.superseded:
return _violation("replaced_not_closed")
if other.phase == "ready" and now > other.valid_until:
return _violation("no_close_at_valid_until")
return Result(True) if ended else _violation("transcript_without_end")
files/contracts/fake_node.py 371 lines · 19.8 KB
"""A Python fake tunnel node for protocol v2 (ALL 2.0): the node side of the contract as a pure state machine.
It is what WP13 (the device's tunnel client) is tested against, and the golden transcripts of contracts/vectors/transcripts.json are its
output: a generator outside this set feeds it the scripted inputs of each scenario and writes down what it did, and the contract test suite
checks that it still does exactly that and that the contract checker (e2e_contracts.check_transcript, the same one the Go and TypeScript
suites run) accepts it. No sockets here: `handle(event)` takes one input event and returns the output events, so a test drives it with a
fake clock, and `FakeNodeServer` (below) is the thin asyncio wrapper that speaks the same lines over real sockets.
Input events (what happens TO the node) Output events (what the node does)
recv {conn, line} a line from a device send {conn, line} a line to a device
cp_reply {conn, status, body} the control plane cp {conn, call, body} a call to the control plane
client_hello {sni, complete, peer_ip, class} tls_alert {sni} the node answers a client with a TLS alert
attach {conn, line} a data connection attach {conn, line, result} ok, or err (with the ERR line in `reply`)
tick {} time passes (send CLOSE expired, ...)
end {} the scenario is over
Every event carries `t`, the node's clock in seconds. Randomness (nonce, stream id, attach secret) is injected so a scenario is deterministic.
"""
from __future__ import annotations
import asyncio
import os
import time
from collections import deque
from typing import Any, Awaitable, Callable
import e2e_contracts as c
# The word the node answers a device with, per reason the contract checker found.
_ERR_FOR = {"malformed_token": "bad_hello", "not_yet_valid": "clock_skew", "stale_proof": "clock_skew", "wrong_audience": "auth", "expired": "auth"}
def err_for(code: str, default: str = "auth") -> str:
return _ERR_FOR.get(code, default)
class FakeNode:
def __init__(self, *, node_id: str, node_url: str, jwks: dict[str, Any], skew: int, randoms: dict[str, list[str]]):
self.node_id = node_id
self.node_url = node_url
self.keys = c.parse_jwks(jwks)
self.skew = skew
# injected randomness first (a scenario is deterministic), fresh random words once it runs out (a live fake node)
self.nonces = deque(randoms.get("nonces", []))
self.stream_ids = deque(randoms.get("stream_ids", []))
self.secrets = deque(randoms.get("secrets", []))
self.conns: dict[str, dict[str, Any]] = {}
self.seen: set[str] = set()
self.streams: dict[str, dict[str, Any]] = {}
# ------------------------------------------------------------------------------------------------------------ plumbing
def _conn(self, name: str) -> dict[str, Any]:
return self.conns.setdefault(name, {"phase": "new", "claims": None, "session": None, "kind": "hello", "nonce": None, "waiting": False,
"host": None, "status": None, "valid_until": 0, "sub": None, "hints": []})
@staticmethod
def _send(t: int, conn: str, line: str) -> dict[str, Any]:
return {"t": t, "conn": conn, "ev": "send", "line": line}
def _err(self, t: int, conn: str, code: str) -> list[dict[str, Any]]:
return [self._send(t, conn, c.format_line("ERR", code=code))]
# ------------------------------------------------------------------------------------------------------------ the entry point
def handle(self, event: dict[str, Any]) -> list[dict[str, Any]]:
t = event["t"]
kind = event["ev"]
if kind == "recv":
return self._recv(t, event["conn"], event["line"])
if kind == "cp_reply":
return self._cp_reply(t, event["conn"], event["status"], event["body"])
if kind == "client_hello":
return self._client_hello(t, event)
if kind == "attach":
return self._attach(t, event)
if kind == "tick":
return self._tick(t)
return []
# ------------------------------------------------------------------------------------------------------------ device lines
def _recv(self, t: int, name: str, line: str) -> list[dict[str, Any]]:
conn = self._conn(name)
try:
message = c.parse_line(line, "d2n")
except c.ProtocolError as error:
return self._err(t, name, error.code)
verb = message["verb"]
if verb in ("HELLO", "RENEW"):
if conn["phase"] != ("new" if verb == "HELLO" else "ready"): # RENEW only on a live connection: not mid-handshake, not after a CLOSE
return self._err(t, name, "bad_command")
result = c.verify_session(message["session"], now=t, audience=c.pol("identity.aud_tunnel"), keys=self.keys, skew=self.skew)
if not result.ok:
return self._err(t, name, err_for(result.error or ""))
if verb == "RENEW" and result.claims["sub"] != conn["sub"]:
return self._err(t, name, "auth")
conn.update(session=message["session"], claims=result.claims, kind="hello" if verb == "HELLO" else "renew", waiting=False)
if verb == "HELLO":
conn["phase"] = "hello"
conn["nonce"] = self.nonces.popleft() if self.nonces else c.b64u_encode(os.urandom(c.pol("protocol.nonce_bytes")))
return [self._send(t, name, c.format_line("CHALLENGE", nonce=conn["nonce"]))]
if verb == "PROOF":
if conn["nonce"] is None:
return self._err(t, name, "bad_command")
result = c.verify_proof(message["proof"], now=t, session_token=conn["session"], session_claims=conn["claims"], nonce=conn["nonce"],
htu=self.node_url, skew=self.skew, seen=self.seen)
conn["nonce"] = None
if not result.ok:
return self._err(t, name, err_for(result.error or "", "bad_proof") if result.error != "replayed_jti" else "bad_proof")
self.seen.add(result.jti or "")
conn["waiting"] = True
body = {"v": 1, "device_id": conn["claims"]["sub"], "session_jti": conn["claims"]["jti"], "reason": conn["kind"], "node": self.node_id}
return [{"t": t, "conn": name, "ev": "cp", "call": "authorize", "body": body}]
if verb == "PING":
return [self._send(t, name, c.format_line("PONG"))]
if verb == "HINT":
conn["hints"] = message["addresses"]
return []
if verb == "CLOSE":
conn["phase"] = "closed"
return []
return self._err(t, name, "bad_command")
def _cp_reply(self, t: int, name: str, status: int, body: dict[str, Any]) -> list[dict[str, Any]]:
conn = self._conn(name)
if not conn["waiting"]:
return []
conn["waiting"] = False
if status != 200:
word = body.get("error") if isinstance(body, dict) else None
known = {"license_inactive": "license_inactive", "device_revoked": "device_revoked", "paused": "paused"}
return self._err(t, name, known.get(word, "control_unavailable" if status >= 500 or status == 429 else "auth"))
if c.SCHEMAS.validate("authorize-response.schema.json", body) or body["device_id"] != conn["claims"]["sub"]:
return self._err(t, name, "control_unavailable")
out: list[dict[str, Any]] = []
ok_body = {"v": 2, "node": self.node_id, "host": body["host"], "status": body["status"], "valid_until": body["valid_until"],
"server_time": body["server_time"], "policy": body["policy"]}
out.append(self._send(t, name, c.format_line("OK", body=ok_body)))
conn.update(phase="ready", host=body["host"], status=body["status"], valid_until=body["valid_until"], sub=conn["claims"]["sub"])
for other_name, other in self.conns.items():
if other_name != name and other["phase"] == "ready" and other["sub"] == conn["sub"]:
other["phase"] = "closed"
out.append(self._send(t, other_name, c.format_line("CLOSE", reason="replaced")))
return out
# ------------------------------------------------------------------------------------------------------------ clients and streams
def _client_hello(self, t: int, event: dict[str, Any]) -> list[dict[str, Any]]:
sni = event["sni"]
if not event["complete"]:
return [{"t": t, "ev": "tls_alert", "sni": sni}]
for name, conn in self.conns.items():
if conn["phase"] == "ready" and conn["host"] == sni and conn["status"] != "paused" and t < conn["valid_until"]:
stream_id = self.stream_ids.popleft() if self.stream_ids else os.urandom(c.pol("protocol.stream_id_bytes")).hex()
secret = self.secrets.popleft() if self.secrets else c.b64u_encode(os.urandom(c.pol("protocol.attach_secret_bytes")))
self.streams[stream_id] = {"secret": secret, "t": t, "consumed": False}
line = c.format_line("OPEN", stream_id=stream_id, secret=secret, peer_ip=event["peer_ip"], **{"class": event["class"]})
return [self._send(t, name, line)]
return [{"t": t, "ev": "tls_alert", "sni": sni}]
def _attach(self, t: int, event: dict[str, Any]) -> list[dict[str, Any]]:
out = {"t": t, "conn": event["conn"], "ev": "attach", "line": event["line"], "result": "err"}
try:
message = c.parse_line(event["line"], "d2n")
except c.ProtocolError:
return [out]
known = self.streams.get(message["stream_id"])
if (known is not None and known["secret"] == message["secret"] and not known["consumed"]
and t - known["t"] <= c.pol("protocol.attach_timeout_s")):
known["consumed"] = True
out["result"] = "ok"
return [out]
def _tick(self, t: int) -> list[dict[str, Any]]:
out: list[dict[str, Any]] = []
for name, conn in self.conns.items():
if conn["phase"] == "ready" and t >= conn["valid_until"]:
conn["phase"] = "closed"
out.append(self._send(t, name, c.format_line("CLOSE", reason="expired")))
return out
INPUT_EVENTS = ("recv", "cp_reply", "client_hello", "attach", "tick", "end")
def split_transcript(events: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
"""(inputs, outputs) of a golden transcript: what was done to the node and what it did. An `attach` is an input without its result."""
inputs: list[dict[str, Any]] = []
outputs: list[dict[str, Any]] = []
for event in events:
if event["ev"] in INPUT_EVENTS:
inputs.append({k: v for k, v in event.items() if k != "result"} if event["ev"] == "attach" else event)
if event["ev"] in ("send", "cp", "tls_alert") or event["ev"] == "attach":
outputs.append(event)
return inputs, outputs
def replay(ctx: dict[str, Any], events: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""Run the fake node over the inputs of a transcript and return the outputs it produced, in order."""
inputs, _ = split_transcript(events)
node = FakeNode(node_id=ctx["node_id"], node_url=ctx["node_url"], jwks=ctx["jwks"], skew=ctx["skew"], randoms=ctx["randoms"])
produced: list[dict[str, Any]] = []
for event in inputs:
produced.extend(node.handle(event))
return produced
def rebuild(ctx: dict[str, Any], inputs: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""The full transcript (inputs and outputs interleaved as they happened) for scripted inputs: what the transcript generator stores."""
node = FakeNode(node_id=ctx["node_id"], node_url=ctx["node_url"], jwks=ctx["jwks"], skew=ctx["skew"], randoms=ctx["randoms"])
events: list[dict[str, Any]] = []
for event in inputs:
outs = node.handle(event)
if event["ev"] == "attach":
events.extend(outs) # the node's own record of the attach carries the result
else:
events.append(event)
events.extend(outs)
return events
# ---------------------------------------------------------------------------------------------------------------------------------- sockets
ControlPlane = Callable[[dict[str, Any]], "tuple[int, dict[str, Any]] | Awaitable[tuple[int, dict[str, Any]]]"]
class OpenedStream:
"""A client stream the fake node opened for a device: `stream_id` and `secret` are what the device's ATTACH must carry; `attached` resolves
to (reader, writer) of the node's end of the data connection once the device attached (bytes written go to the device, bytes read come from it)."""
def __init__(self, stream_id: str, secret: str, peer_ip: str, cls: str, attached: "asyncio.Future[tuple[asyncio.StreamReader, asyncio.StreamWriter]]"):
self.stream_id, self.secret, self.peer_ip, self.cls, self.attached = stream_id, secret, peer_ip, cls, attached
class FakeNodeServer:
"""The fake node over real sockets: a TCP (or TLS, with `ssl_context`) listener that speaks protocol v2 lines, the state machine above behind it.
`control_plane(body)` answers the node's authorize call (it is given the request body, returns (status, body); sync or async). `clock()` is the
node's time in whole seconds. `open_stream(sni, ...)` plays a client that connected to :443 with that SNI: when the node opens a stream for it,
the result says so and its `attached` future resolves when the device dials back with the right ATTACH. Nothing here is used by an installed ALL.
"""
def __init__(self, *, node_id: str, node_url: str, jwks: dict[str, Any], control_plane: ControlPlane, skew: int | None = None,
clock: Callable[[], int] | None = None, tick_interval: float = 1.0):
self.node = FakeNode(node_id=node_id, node_url=node_url, jwks=jwks, skew=c.pol("authorization.node_skew_s") if skew is None else skew, randoms={})
self.control_plane = control_plane
self.clock = clock or (lambda: int(time.time()))
self.tick_interval = tick_interval
self.writers: dict[str, asyncio.StreamWriter] = {}
self.lines: list[tuple[str, str, str]] = [] # (direction, connection, line): what crossed the wire, for the tests
self._opened: dict[str, OpenedStream] = {}
self._open_writers: set[asyncio.StreamWriter] = set() # every accepted connection, control and data
self._count = 0
self._server: asyncio.AbstractServer | None = None
self._ticker: asyncio.Task[None] | None = None
self.port = 0
async def start(self, host: str = "127.0.0.1", port: int = 0, ssl_context: Any = None) -> int:
limit = c.pol("protocol.statement_line_max_bytes") + 16
self._server = await asyncio.start_server(self._serve, host, port, ssl=ssl_context, limit=limit)
self.port = self._server.sockets[0].getsockname()[1]
self._ticker = asyncio.create_task(self._tick_loop())
return self.port
async def stop(self) -> None:
if self._ticker:
self._ticker.cancel()
for writer in list(self._open_writers):
writer.close()
if self._server:
self._server.close()
try:
await asyncio.wait_for(self._server.wait_closed(), 2)
except asyncio.TimeoutError:
pass
async def _tick_loop(self) -> None:
while True:
await asyncio.sleep(self.tick_interval)
await self.tick()
async def tick(self) -> None:
await self._dispatch(self.node.handle({"t": self.clock(), "ev": "tick"}))
def drop_control(self, name: str) -> None:
"""Cut one control connection (a node restart, a network drop)."""
writer = self.writers.pop(name, None)
if writer:
writer.close()
async def open_stream(self, sni: str, *, peer_ip: str = "203.0.113.9", cls: str = "anon", complete: bool = True) -> OpenedStream | None:
"""A client with this SNI arrives. None when the node would send a TLS alert (unknown name, paused, expired, incomplete ClientHello)."""
outputs = self.node.handle({"t": self.clock(), "ev": "client_hello", "sni": sni, "complete": complete, "peer_ip": peer_ip, "class": cls})
opened = None
for event in outputs:
if event["ev"] == "send" and event["line"].startswith("OPEN "):
message = c.parse_line(event["line"], "n2d")
opened = OpenedStream(message["stream_id"], message["secret"], peer_ip, cls, asyncio.get_running_loop().create_future())
self._opened[opened.stream_id] = opened
await self._dispatch(outputs)
return opened
async def _dispatch(self, outputs: list[dict[str, Any]]) -> None:
pending = list(outputs)
while pending:
event = pending.pop(0)
if event["ev"] == "send":
writer = self.writers.get(event["conn"])
line = event["line"]
self.lines.append(("send", event["conn"], line))
if writer is None:
continue
writer.write(line.encode("ascii") + b"\n")
await writer.drain()
refused = line.startswith("ERR ") and self.node.conns[event["conn"]]["phase"] == "new"
if line.startswith("CLOSE ") or refused:
writer.close()
self.writers.pop(event["conn"], None)
elif event["ev"] == "cp":
reply = self.control_plane(event["body"])
if asyncio.iscoroutine(reply):
reply = await reply
status, body = reply
pending.extend(self.node.handle({"t": self.clock(), "ev": "cp_reply", "conn": event["conn"], "status": status, "body": body}))
async def _serve(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None:
self._count += 1
name = f"c{self._count}"
self._open_writers.add(writer)
try:
first = await asyncio.wait_for(self._read(reader), c.pol("protocol.first_line_timeout_s"))
if first is None:
return
if first.startswith("ATTACH "):
await self._data(name, first, reader, writer)
return
self.writers[name] = writer
line: str | None = first
while line is not None:
self.lines.append(("recv", name, line))
await self._dispatch(self.node.handle({"t": self.clock(), "ev": "recv", "conn": name, "line": line}))
if name not in self.writers:
break
line = await self._read(reader)
except (asyncio.TimeoutError, ConnectionError, asyncio.IncompleteReadError, asyncio.LimitOverrunError):
pass
finally:
self._open_writers.discard(writer)
if self.writers.get(name) is writer:
self.writers.pop(name, None)
writer.close()
@staticmethod
async def _read(reader: asyncio.StreamReader) -> str | None:
try:
raw = await reader.readuntil(b"\n")
except asyncio.IncompleteReadError:
return None
return raw[:-1].decode("ascii", errors="replace")
async def _data(self, name: str, line: str, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None:
outputs = self.node.handle({"t": self.clock(), "ev": "attach", "conn": name, "line": line})
if outputs[0]["result"] != "ok":
writer.write(b"ERR auth\n")
await writer.drain()
writer.close()
return
message = c.parse_line(line, "d2n")
opened = self._opened.get(message["stream_id"])
if opened is not None and not opened.attached.done():
opened.attached.set_result((reader, writer))
files/contracts/policy.json 376 lines · 12.6 KB
{
"$comment": "ALL 2.0 policy table: the ONE place where a number, a window or a limit of the end-to-end path is written down (plan section 0 and 7.1). The tunnel node (Go), the control plane (TypeScript) and the device (Python) read it; nobody copies a value into code (the contract generator lint fails the suite when one does). Units are in the key suffix: _s seconds, _bytes, _days, _pct percent. Keys listed in `provisional` are the plan's initial values: they are retuned after G0 and G1 and ONLY here.",
"version": 1,
"provisional": [
"timing.stream_idle_s",
"timing.stream_max_age_s",
"timing.ping_interval_s",
"timing.dead_peer_s",
"node_limits.per_ip_concurrent",
"node_limits.per_ip_new_per_s",
"node_limits.label_streams_max",
"node_limits.label_anon_slice",
"node_limits.label_hint_slice",
"node_limits.global_streams_max",
"node_limits.openai_reserved_pct",
"node_limits.openai_per_ip_factor",
"node_limits.device_per_ip_factor",
"node_limits.ip_table_max",
"node_limits.unknown_sni_per_ip",
"node_limits.unknown_sni_window_s",
"node_limits.unknown_sni_ban_s",
"node_limits.label_new_per_s",
"node_limits.label_new_burst",
"node_limits.hint_addresses_per_label",
"terminator.endpoint_budgets",
"terminator.max_streams",
"budget.borrow_from",
"budget.defer_jitter_s",
"budget.dns_calls_per_reservation",
"budget.reservations_per_label_per_day",
"budget.trial_per_ip_per_window",
"budget.trial_per_email_domain_per_window",
"budget.trial_per_mailbox_per_window",
"budget.lapsed_lift_after_s",
"budget.abandons_per_label_per_window",
"monitoring.lapsed_held_alert",
"monitoring.jwks_age_alert_s",
"monitoring.authorize_window_s",
"monitoring.authorize_fail_alert_pct",
"monitoring.authorize_fail_min_calls",
"monitoring.criteria_min_coverage_pct",
"migration.connected_now_s",
"migration.migrate_floor_pct",
"migration.lend_horizon_s",
"migration.ineligible_hold_s",
"migration.retire.min_days_since_ready",
"migration.retire.sustain_days",
"migration.retire.node_availability_permille_min",
"migration.retire.authorize_failure_pct_max",
"migration.retire.migrated_pct_min",
"migration.retire.open_kill_events_max",
"dns_failover.remove_after_failures",
"dns_failover.restore_after_successes",
"dns_failover.min_change_interval_s",
"concierge.turns_per_session",
"concierge.mutations_per_session"
],
"protocol": {
"version": 2,
"alpn": "all-tunnel/2",
"line_max_bytes": 8192,
"open_line_max_bytes": 192,
"statement_line_max_bytes": 65552,
"jwt_max_bytes": 4096,
"first_line_timeout_s": 10,
"attach_timeout_s": 10,
"stream_id_bytes": 16,
"attach_secret_bytes": 32,
"nonce_bytes": 16,
"hint_max_addresses": 16,
"proof_htm": "GET",
"proof_htu_path": "/all-tunnel/2",
"node_assertion_header": "x-all-node-assertion",
"source_classes": ["openai", "hint", "anon"],
"close_reasons": ["replaced", "revoked", "expired", "released", "killed", "paused", "drain", "idle", "protocol", "shutdown"],
"error_codes": [
"bad_hello", "bad_command", "unsupported_version", "bad_proof", "auth", "clock_skew", "unknown_stream", "bad_attach",
"license_inactive", "device_revoked", "paused", "busy", "control_unavailable", "line_too_long"
]
},
"timing": {
"ping_interval_s": 20,
"dead_peer_s": 45,
"stream_idle_s": 1800,
"stream_max_age_s": 14400,
"half_close_s": 30,
"backoff_min_s": 1,
"backoff_max_s": 60,
"renew_before_valid_until_s": 300,
"drain_quiet_s": 1200,
"feed_poll_s": 15,
"feed_apply_deadline_s": 20,
"reconnect_after_wake_s": 5
},
"authorization": {
"valid_until_paid_s": 14400,
"valid_until_trial_s": 900,
"valid_until_gated_s": 900,
"grace_paid_s": 900,
"authorize_max_concurrent": 50,
"session_ttl_s": 300,
"proof_ttl_s": 60,
"node_assertion_ttl_s": 60,
"node_skew_s": 5,
"node_skew_max_s": 30,
"access_ttl_paid_s": 14400,
"access_ttl_trial_s": 600,
"access_ttl_paywall_s": 600,
"access_future_slack_s": 60,
"stripe_status_cache_s": 120,
"last_seen_write_interval_s": 300
},
"gated": {
"bytes_per_day": 10485760,
"streams_per_day": 200,
"cutoff_days": 14
},
"license": {
"revoked_notice_days": 7
},
"identity": {
"issuer": "https://api.all-mcp.fyi",
"aud_tunnel": "all-tunnel",
"aud_access": "all-device-access",
"aud_node_assertion": "all-tunnel-control",
"label_length": 20,
"label_alphabet": "abcdefghijklmnopqrstuvwxyz234567",
"label_regex": "^[a-z2-7]{20}$",
"fqdn_regex": "^[a-z2-7]{20}\\.(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\\.)+[a-z]{2,63}$",
"fqdn_max_length": 253,
"tombstone_idle_months": 12,
"device_id_regex": "^[a-f0-9]{32}$",
"stream_id_regex": "^[0-9a-f]{32}$",
"attach_secret_regex": "^[A-Za-z0-9_-]{43}$",
"nonce_regex": "^[A-Za-z0-9_-]{22,64}$",
"jti_regex": "^[A-Za-z0-9_-]{16,128}$",
"kid_regex": "^[A-Za-z0-9._-]{1,128}$"
},
"jwks": {
"cache_s": 300,
"max_keys": 16,
"stale_fail_closed_s": 1800
},
"node_limits": {
"client_hello_deadline_s": 5,
"per_ip_concurrent": 16,
"per_ip_new_per_s": 10,
"ipv6_prefix_bits": 64,
"label_streams_max": 64,
"label_anon_slice": 16,
"label_hint_slice": 48,
"hint_ttl_s": 900,
"global_streams_max": 2048,
"openai_reserved_pct": 25,
"openai_prefix_refresh_s": 86400,
"openai_prefix_max": 2048,
"openai_prefix_min_v4_bits": 16,
"openai_prefix_min_v6_bits": 32,
"openai_prefix_body_max_bytes": 1048576,
"openai_per_ip_factor": 8,
"device_per_ip_factor": 8,
"ip_table_max": 65536,
"unknown_sni_per_ip": 20,
"unknown_sni_window_s": 60,
"unknown_sni_ban_s": 600,
"label_new_per_s": 20,
"label_new_burst": 32,
"hint_addresses_per_label": 64,
"tcp_keepalive_s": 30,
"max_unauthenticated_control": 256
},
"terminator": {
"tls_handshake_s": 10,
"request_head_s": 10,
"request_head_max_bytes": 16384,
"max_streams": 64,
"endpoint_budgets": {
"register": {"per_ip_per_min": 20},
"authorize": {"per_ip_per_min": 60},
"token": {"per_ip_per_min": 120},
"pair": {"per_ip_per_min": 30},
"mcp_anonymous": {"per_ip_per_min": 60}
},
"tls_min_version": "TLSv1.3",
"tls_alpn": "http/1.1"
},
"surface": {
"body_max_bytes": 16777216,
"headers_max": 64,
"header_value_max_bytes": 32768,
"keep_alive_s": 75,
"limiter_lru_max": 4096,
"never_authorized_client_evict_hours": 24,
"identity_path": "/__all/surface/v1/proof",
"identity_challenge_header": "x-all-challenge",
"identity_mac_header": "x-all-challenge-mac",
"identity_proof_header": "x-all-surface-proof"
},
"pairing": {
"window_s": 600,
"code_ttl_e2e_s": 600,
"code_ttl_local_s": 86400,
"max_codes_local": 16,
"bad_codes_burn": 5
},
"certificates": {
"profile": "tlsserver",
"profile_fallback": "classic",
"profile_days": 45,
"renew_backstop_divisor": 3,
"ari_checks_per_day": 2,
"min_attempt_gap_s": 21600,
"max_issuances_per_day": 2,
"txt_ttl_s": 60,
"txt_value_length": 43,
"ct_check_interval_s": 86400,
"ct_degraded_after_s": 259200,
"wait_after_install_s": 300
},
"dns": {
"a_record_ttl_s": 60,
"devices_caa": ["0 issue \"letsencrypt.org; validationmethods=dns-01\"", "0 issuewild \";\""],
"node_caa": ["0 issue \"letsencrypt.org\""]
},
"budget": {
"window_days": 7,
"first_issuances_total": 40,
"classes": {"paid": 20, "trial": 12, "flex": 8},
"borrow_only_unreserved_hours": 24,
"per_label_per_window": 3,
"per_license_per_day": 2,
"reservation_ttl_s": 900,
"alert_pct": 70,
"borrow_from": {"paid": ["flex", "trial"], "trial": ["flex"], "flex": ["trial"]},
"defer_jitter_s": 7200,
"dns_calls_per_reservation": 8,
"reservations_per_label_per_day": 8,
"lapsed_lift_after_s": 93600,
"abandons_per_label_per_window": 3,
"trial_per_ip_per_window": 3,
"trial_per_email_domain_per_window": 3,
"trial_per_mailbox_per_window": 2,
"free_mail_domains": [
"gmail.com", "googlemail.com", "outlook.com", "hotmail.com", "live.com", "msn.com", "yahoo.com", "icloud.com", "me.com", "aol.com",
"proton.me", "protonmail.com", "gmx.com", "yandex.com", "mail.com", "zoho.com"
]
},
"monitoring": {
"node_probe_interval_s": 300,
"probe_failures_to_alert": 2,
"node_cert_alert_days": [14, 5],
"ct_watch_interval_s": 900,
"lapsed_held_alert": 3,
"device_cert_expiring_days": 10,
"fleet_e2e_retention_s": 2592000,
"fleet_e2e_window_s": 86400,
"fleet_e2e_window_dedup_s": 172800,
"fleet_e2e_window_dedup_max": 8,
"jwks_age_alert_s": 900,
"authorize_bucket_s": 300,
"authorize_window_s": 900,
"authorize_fail_alert_pct": 10,
"authorize_fail_min_calls": 20,
"authorize_tally_keep_s": 259200,
"criteria_min_coverage_pct": 90
},
"dns_failover": {
"remove_after_failures": 3,
"restore_after_successes": 5,
"min_change_interval_s": 600,
"min_in_rotation": 1,
"record_ttl_s": 60,
"hold_default_s": 7200,
"hold_max_s": 86400,
"log_entries": 50
},
"release": {
"proof_max_age_h": 72
},
"migration": {
"legacy_close_after_days": 30,
"delete_old_connector_after_days": 7,
"connected_now_s": 300,
"wave_hash_prefix": "all-e2e-wave-v1/",
"default_wave_pct": 100,
"min_lead_s": 60,
"min_version": "2.0.0",
"ready_report_max_age_s": 86400,
"verdict_cache_s": 300,
"migrate_floor_pct": 50,
"lend_horizon_s": 86400,
"ineligible_hold_s": 604800,
"enroll_refresh_min_s": 900,
"enroll_refresh_max_s": 1800,
"kill_apply_deadline_s": 3600,
"blocked_after_failures": 3,
"blocked_retry_s": 21600,
"retire": {
"$note": "The owner's figures of 2026-10-27 for retiring the old relay: every one of the last 30 complete UTC days has the nodes up at least 99.5 percent (_permille is thousandths: 995 is 99.5 percent, because nothing in this file is a fraction) and authorize failures at most 1 percent, and at least 95 percent of the eligible computers that enrolled have migrated. Even then the relay is retired only on the owner's explicit order.",
"min_days_since_ready": 14,
"sustain_days": 30,
"node_availability_permille_min": 995,
"authorize_failure_pct_max": 1,
"migrated_pct_min": 95,
"open_kill_events_max": 0
},
"selfcheck": {
"timeout_s": 10,
"min_interval_s": 60
}
},
"keep_awake": {
"hold_cap_s": 28800,
"release_grace_s": 5,
"reconcile_s": 2
},
"concierge": {
"$comment": "Robi concierge (owner order of 2026-10-27): a customer who proves an email with a one-time code gets a short session in which Robi can fix that customer's own account. Refund rules are the owner's exact ones and are in the Terms: a refund is refused when ALL ran more than max_actions_in_window actions in the refund window (monthly: the 3 days since the charge; lifetime: the 5 days since purchase). Not provisional: the owner set the number.",
"session_s": 1800,
"code_sends_per_net_hour": 6,
"code_checks_per_net_hour": 30,
"chat_turns_per_net_hour": 60,
"turns_per_day_total": 2000,
"voice_calls_per_session": 80,
"summary_chars_max": 800,
"turns_per_session": 40,
"tool_rounds_per_turn": 6,
"mutations_per_session": 8,
"history_messages_max": 16,
"user_chars_max": 1400,
"assistant_chars_max": 6000,
"history_chars_max": 24000,
"output_tokens_max": 1024,
"model_deadline_s": 25,
"audit_retention_days": 400,
"refund": {
"monthly_window_h": 72,
"lifetime_window_days": 5,
"max_actions_in_window": 100,
"confirm_ttl_s": 600
}
},
"enums": {
"authorize_status": ["active", "gated", "paused"],
"statement_mode": ["active", "paywall"],
"budget_class": ["paid", "trial", "flex"],
"acme_purpose": ["first", "renewal"],
"reserve_status": ["granted", "deferred"],
"feed_event_type": ["revoke", "pause", "resume", "license_end", "release", "kill", "kill_clear"],
"e2e_state": ["off", "unenrolled", "waiting", "issuing", "connecting", "ready", "degraded", "clock_skew", "compat_only", "killed"],
"e2e_cohort": ["paid", "trial", "flex", "legacy"],
"ineligible_reason": ["update_not_in_plan", "legacy_credential", "unsupported_version"],
"selfcheck_reason": ["chain", "expired", "spki", "name", "tls_version", "healthz", "unreachable", "timeout"]
}
}
files/contracts/schemas/acme-abandon-request.schema.json 12 lines · 909 B
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://all-mcp.fyi/contracts/v2/acme-abandon-request.schema.json",
"title": "POST /v1/devices/acme/abandon: the computer's order for this reservation ended without a finalized certificate (a validation that failed, a timeout before the CSR was sent). The broker frees the slot a lapsed reservation would otherwise keep for the whole window, at most policy.budget.abandons_per_label_per_window times per label and window; what it frees in that case it no longer counts for the label either. A computer that did send the CSR never calls this: a certificate may exist, and only the CT reconciliation can lift that hold.",
"type": "object",
"required": ["v", "reservation_id"],
"additionalProperties": false,
"properties": {
"v": {"const": 1},
"reservation_id": {"$ref": "common.schema.json#/$defs/reservation_id"}
}
}
files/contracts/schemas/acme-budget-response.schema.json 38 lines · 1.9 KB
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://all-mcp.fyi/contracts/v2/acme-budget-response.schema.json",
"title": "GET /v1/commerce/acme/budget, commerce key, read only: the first-issuance budget of the rolling window as the broker sees it now (the 70 percent alert of the monitor and the wave calendar read it). used = confirmed first issuances still inside the window plus the open reservations; reserved is the part of used that is not yet confirmed; quota is the class's own share of policy.budget.first_issuances_total; a class above its quota is borrowing. lapsed = how many of the used slots are held by a reservation that ended unconfirmed after a TXT was set (see acme-lapsed-response): the alert of the monitor on a systemic failure of the orders.",
"type": "object",
"required": ["ok", "v", "server_time", "window_days", "total", "classes", "lapsed"],
"additionalProperties": false,
"properties": {
"ok": {"const": true},
"v": {"const": 1},
"server_time": {"$ref": "common.schema.json#/$defs/epoch_s"},
"window_days": {"type": "integer", "minimum": 1, "maximum": 365},
"lapsed": {"$ref": "common.schema.json#/$defs/count"},
"total": {"$ref": "#/$defs/share"},
"classes": {
"type": "object",
"required": ["paid", "trial", "flex"],
"additionalProperties": false,
"properties": {
"paid": {"$ref": "#/$defs/share"},
"trial": {"$ref": "#/$defs/share"},
"flex": {"$ref": "#/$defs/share"}
}
}
},
"$defs": {
"share": {
"type": "object",
"required": ["quota", "used", "reserved"],
"additionalProperties": false,
"properties": {
"quota": {"$ref": "common.schema.json#/$defs/count"},
"used": {"$ref": "common.schema.json#/$defs/count"},
"reserved": {"$ref": "common.schema.json#/$defs/count"}
}
}
}
}
files/contracts/schemas/acme-confirm-request.schema.json 15 lines · 726 B
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://all-mcp.fyi/contracts/v2/acme-confirm-request.schema.json",
"title": "POST /v1/devices/acme/confirm. The budget is consumed here, not at reserve. spki_sha256 must be the one reserved.",
"type": "object",
"required": ["v", "reservation_id", "spki_sha256", "not_before", "not_after"],
"additionalProperties": false,
"properties": {
"v": {"const": 1},
"reservation_id": {"$ref": "common.schema.json#/$defs/reservation_id"},
"spki_sha256": {"$ref": "common.schema.json#/$defs/sha256_b64url"},
"not_before": {"$ref": "common.schema.json#/$defs/epoch_s"},
"not_after": {"$ref": "common.schema.json#/$defs/epoch_s"}
}
}
files/contracts/schemas/acme-confirm-response.schema.json 13 lines · 428 B
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://all-mcp.fyi/contracts/v2/acme-confirm-response.schema.json",
"title": "200 answer of confirm and of release.",
"type": "object",
"required": ["ok", "v", "server_time"],
"additionalProperties": false,
"properties": {
"ok": {"const": true},
"v": {"const": 1},
"server_time": {"$ref": "common.schema.json#/$defs/epoch_s"}
}
}
files/contracts/schemas/acme-dns-request.schema.json 28 lines · 1.1 KB
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://all-mcp.fyi/contracts/v2/acme-dns-request.schema.json",
"title": "POST /v1/devices/acme/dns. Needs a live reservation. The only name that can ever be written is _acme-challenge.<the caller's own host>, type TXT, a 43-character value, TTL policy.certificates.txt_ttl_s: nothing in the request names a record.",
"oneOf": [
{
"type": "object",
"required": ["v", "reservation_id", "action", "value"],
"additionalProperties": false,
"properties": {
"v": {"const": 1},
"reservation_id": {"$ref": "common.schema.json#/$defs/reservation_id"},
"action": {"const": "set"},
"value": {"type": "string", "pattern": "^[A-Za-z0-9_-]{43}$"}
}
},
{
"type": "object",
"required": ["v", "reservation_id", "action"],
"additionalProperties": false,
"properties": {
"v": {"const": 1},
"reservation_id": {"$ref": "common.schema.json#/$defs/reservation_id"},
"action": {"const": "clear"}
}
}
]
}
files/contracts/schemas/acme-dns-response.schema.json 15 lines · 522 B
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://all-mcp.fyi/contracts/v2/acme-dns-response.schema.json",
"title": "200 answer of acme dns (set and clear).",
"type": "object",
"required": ["ok", "v", "name", "type", "ttl"],
"additionalProperties": false,
"properties": {
"ok": {"const": true},
"v": {"const": 1},
"name": {"$ref": "common.schema.json#/$defs/acme_name"},
"type": {"const": "TXT"},
"ttl": {"type": "integer", "minimum": 1, "maximum": 3600}
}
}
files/contracts/schemas/acme-expected-response.schema.json 33 lines · 2.0 KB
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://all-mcp.fyi/contracts/v2/acme-expected-response.schema.json",
"title": "GET /v1/commerce/acme/expected?after=<cursor>&limit=<n>, commerce key, read only: every key the broker has recorded for any label, in the order it was recorded, so that a CT watcher can tell an expected certificate from a mis-issued one. state reserved = recorded before the order and not yet confirmed (a certificate for it may exist), issued = confirmed, expired = never confirmed within the reservation lifetime (a certificate for it may still exist), released = replaced by a newer reservation before use. retired = the label is a tombstone. next is the cursor of the following page, null at the end.",
"type": "object",
"required": ["ok", "v", "server_time", "entries", "next"],
"additionalProperties": false,
"properties": {
"ok": {"const": true},
"v": {"const": 1},
"server_time": {"$ref": "common.schema.json#/$defs/epoch_s"},
"entries": {
"type": "array",
"maxItems": 500,
"items": {
"type": "object",
"required": ["label", "host", "spki_sha256", "state", "recorded_at", "not_before", "not_after", "retired"],
"additionalProperties": false,
"properties": {
"label": {"$ref": "common.schema.json#/$defs/label"},
"host": {"$ref": "common.schema.json#/$defs/fqdn"},
"spki_sha256": {"$ref": "common.schema.json#/$defs/sha256_b64url"},
"state": {"type": "string", "enum": ["reserved", "issued", "expired", "released"]},
"recorded_at": {"$ref": "common.schema.json#/$defs/epoch_s"},
"not_before": {"oneOf": [{"type": "null"}, {"$ref": "common.schema.json#/$defs/epoch_s"}]},
"not_after": {"oneOf": [{"type": "null"}, {"$ref": "common.schema.json#/$defs/epoch_s"}]},
"retired": {"type": "boolean"}
}
}
},
"next": {"oneOf": [{"type": "null"}, {"$ref": "common.schema.json#/$defs/count"}]}
}
}
files/contracts/schemas/acme-lapsed-response.schema.json 29 lines · 1.6 KB
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://all-mcp.fyi/contracts/v2/acme-lapsed-response.schema.json",
"title": "GET /v1/commerce/acme/lapsed?limit=<n>, a read key: the first issuances whose reservation ended unconfirmed after the computer had set a TXT, so that the budget keeps their slot for the whole window (an order may have been finalized and the CA may have issued a certificate nobody told the broker about). total counts all of them; entries are the ones at least policy.budget.lapsed_lift_after_s past the end of their reservation, which is the earliest a CT reconciliation may lift them (the CT logs need that long to show a certificate), oldest first, 100 at most. ended_at is when the reservation ran out.",
"type": "object",
"required": ["ok", "v", "server_time", "total", "entries"],
"additionalProperties": false,
"properties": {
"ok": {"const": true},
"v": {"const": 1},
"server_time": {"$ref": "common.schema.json#/$defs/epoch_s"},
"total": {"$ref": "common.schema.json#/$defs/count"},
"entries": {
"type": "array",
"maxItems": 100,
"items": {
"type": "object",
"required": ["label", "host", "spki_sha256", "ended_at"],
"additionalProperties": false,
"properties": {
"label": {"$ref": "common.schema.json#/$defs/label"},
"host": {"$ref": "common.schema.json#/$defs/fqdn"},
"spki_sha256": {"$ref": "common.schema.json#/$defs/sha256_b64url"},
"ended_at": {"$ref": "common.schema.json#/$defs/epoch_s"}
}
}
}
}
}
files/contracts/schemas/acme-lift-request.schema.json 12 lines · 872 B
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://all-mcp.fyi/contracts/v2/acme-lift-request.schema.json",
"title": "POST /v1/commerce/acme/lift, the commerce key (a write): the CT reconciliation found no certificate for these keys in the logs, so the slot their lapsed reservation keeps is released. The broker lifts only a key that is a lapsed first reservation at least policy.budget.lapsed_lift_after_s past its end; any other key in the list is left as it is. The key stays recorded and expected for ever, so a certificate that turns up later is still not foreign.",
"type": "object",
"required": ["v", "spki_sha256"],
"additionalProperties": false,
"properties": {
"v": {"const": 1},
"spki_sha256": {"type": "array", "minItems": 1, "maxItems": 100, "items": {"$ref": "common.schema.json#/$defs/sha256_b64url"}}
}
}
files/contracts/schemas/acme-lift-response.schema.json 14 lines · 545 B
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://all-mcp.fyi/contracts/v2/acme-lift-response.schema.json",
"title": "200 answer of POST /v1/commerce/acme/lift: how many of the keys had their slot released.",
"type": "object",
"required": ["ok", "v", "server_time", "lifted"],
"additionalProperties": false,
"properties": {
"ok": {"const": true},
"v": {"const": 1},
"server_time": {"$ref": "common.schema.json#/$defs/epoch_s"},
"lifted": {"$ref": "common.schema.json#/$defs/count"}
}
}
files/contracts/schemas/acme-release-request.schema.json 12 lines · 536 B
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://all-mcp.fyi/contracts/v2/acme-release-request.schema.json",
"title": "POST /v1/devices/acme/release: tombstones the label and clears its TXT (uninstall and reinstall). The control plane does the same by itself when the license is released or revoked.",
"type": "object",
"required": ["v", "reason"],
"additionalProperties": false,
"properties": {
"v": {"const": 1},
"reason": {"type": "string", "enum": ["uninstall", "reinstall"]}
}
}
files/contracts/schemas/acme-reserve-request.schema.json 13 lines · 739 B
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://all-mcp.fyi/contracts/v2/acme-reserve-request.schema.json",
"title": "POST /v1/devices/acme/reserve. The device sends the SHA-256 of the SubjectPublicKeyInfo of the key it just made BEFORE it opens an ACME order; the broker records it first (the registry the CT watchers compare against). purpose first counts against the class budget when confirmed; renewal never does.",
"type": "object",
"required": ["v", "spki_sha256", "purpose"],
"additionalProperties": false,
"properties": {
"v": {"const": 1},
"spki_sha256": {"$ref": "common.schema.json#/$defs/sha256_b64url"},
"purpose": {"type": "string", "enum": ["first", "renewal"]}
}
}
files/contracts/schemas/acme-reserve-response.schema.json 33 lines · 1.4 KB
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://all-mcp.fyi/contracts/v2/acme-reserve-response.schema.json",
"title": "200 answer of reserve. Over budget is NEVER an error: status deferred carries enroll_after (with jitter) and the device keeps whatever certificate it has. A granted reservation that is not confirmed within policy.budget.reservation_ttl_s frees its slot.",
"oneOf": [
{
"type": "object",
"required": ["ok", "v", "status", "reservation_id", "class", "expires_at", "server_time"],
"additionalProperties": false,
"properties": {
"ok": {"const": true},
"v": {"const": 1},
"status": {"const": "granted"},
"reservation_id": {"$ref": "common.schema.json#/$defs/reservation_id"},
"class": {"$ref": "common.schema.json#/$defs/budget_class"},
"expires_at": {"$ref": "common.schema.json#/$defs/epoch_s"},
"server_time": {"$ref": "common.schema.json#/$defs/epoch_s"}
}
},
{
"type": "object",
"required": ["ok", "v", "status", "enroll_after", "server_time"],
"additionalProperties": false,
"properties": {
"ok": {"const": true},
"v": {"const": 1},
"status": {"const": "deferred"},
"enroll_after": {"$ref": "common.schema.json#/$defs/epoch_s"},
"server_time": {"$ref": "common.schema.json#/$defs/epoch_s"}
}
}
]
}
files/contracts/schemas/authorize-request.schema.json 15 lines · 779 B
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://all-mcp.fyi/contracts/v2/authorize-request.schema.json",
"title": "POST /v1/tunnel/authorize, node to control plane (authenticated by the node assertion header, never by anything a device sends). The node never sends a host name: the control plane answers it from the device id.",
"type": "object",
"required": ["v", "device_id", "session_jti", "reason", "node"],
"additionalProperties": false,
"properties": {
"v": {"const": 1},
"device_id": {"$ref": "common.schema.json#/$defs/device_id"},
"session_jti": {"$ref": "common.schema.json#/$defs/jti"},
"reason": {"type": "string", "enum": ["hello", "renew"]},
"node": {"$ref": "common.schema.json#/$defs/node_id"}
}
}
files/contracts/schemas/authorize-response.schema.json 25 lines · 1.4 KB
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://all-mcp.fyi/contracts/v2/authorize-response.schema.json",
"title": "200 answer of POST /v1/tunnel/authorize. valid_until is paid up to 14400 s, trial and gated at most 900 s and never beyond the trial's end (policy.authorization); status paused keeps the connection but no client stream is opened.",
"type": "object",
"required": ["ok", "v", "device_id", "label", "host", "status", "plan", "valid_until", "server_time", "policy", "nodes", "e2e_wave", "e2e_kill", "copy_e2ee"],
"additionalProperties": false,
"properties": {
"ok": {"const": true},
"v": {"const": 1},
"device_id": {"$ref": "common.schema.json#/$defs/device_id"},
"label": {"$ref": "common.schema.json#/$defs/label"},
"host": {"$ref": "common.schema.json#/$defs/fqdn"},
"status": {"$ref": "common.schema.json#/$defs/authorize_status"},
"plan": {"type": "string", "enum": ["paid", "trial"]},
"valid_until": {"$ref": "common.schema.json#/$defs/epoch_s"},
"server_time": {"$ref": "common.schema.json#/$defs/epoch_s"},
"policy": {"$ref": "ok-body.schema.json#/properties/policy"},
"nodes": {"$ref": "common.schema.json#/$defs/nodes"},
"access": {"$ref": "common.schema.json#/$defs/jwt"},
"e2e_wave": {"type": "integer", "minimum": 0, "maximum": 100},
"e2e_kill": {"type": "boolean"},
"copy_e2ee": {"type": "boolean"}
}
}
files/contracts/schemas/cert-status-response.schema.json 46 lines · 1.8 KB
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://all-mcp.fyi/contracts/v2/cert-status-response.schema.json",
"title": "GET /v1/devices/acme/status: what the broker holds for the caller's label (no secrets, never a TXT value).",
"type": "object",
"required": ["ok", "v", "label", "host", "class", "reservation", "issuances", "enroll_after", "server_time"],
"additionalProperties": false,
"properties": {
"ok": {"const": true},
"v": {"const": 1},
"label": {"$ref": "common.schema.json#/$defs/label"},
"host": {"$ref": "common.schema.json#/$defs/fqdn"},
"class": {"$ref": "common.schema.json#/$defs/budget_class"},
"reservation": {
"oneOf": [
{"type": "null"},
{
"type": "object",
"required": ["reservation_id", "spki_sha256", "expires_at"],
"additionalProperties": false,
"properties": {
"reservation_id": {"$ref": "common.schema.json#/$defs/reservation_id"},
"spki_sha256": {"$ref": "common.schema.json#/$defs/sha256_b64url"},
"expires_at": {"$ref": "common.schema.json#/$defs/epoch_s"}
}
}
]
},
"issuances": {
"type": "array",
"maxItems": 64,
"items": {
"type": "object",
"required": ["spki_sha256", "not_before", "not_after"],
"additionalProperties": false,
"properties": {
"spki_sha256": {"$ref": "common.schema.json#/$defs/sha256_b64url"},
"not_before": {"$ref": "common.schema.json#/$defs/epoch_s"},
"not_after": {"$ref": "common.schema.json#/$defs/epoch_s"}
}
}
},
"enroll_after": {"oneOf": [{"type": "null"}, {"$ref": "common.schema.json#/$defs/epoch_s"}]},
"server_time": {"$ref": "common.schema.json#/$defs/epoch_s"}
}
}
files/contracts/schemas/common.schema.json 208 lines · 4.4 KB
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://all-mcp.fyi/contracts/v2/common.schema.json",
"title": "Shared definitions of the ALL 2.0 contracts. Every pattern equals the matching key of contracts/policy.json (the contract test suite pins it).",
"$defs": {
"epoch_s": {
"type": "integer",
"minimum": 1,
"maximum": 4102444800
},
"epoch_ms": {
"type": "integer",
"minimum": 1,
"maximum": 8640000000000000
},
"count": {
"type": "integer",
"minimum": 0,
"maximum": 9007199254740991
},
"device_id": {
"type": "string",
"pattern": "^[a-f0-9]{32}$"
},
"label": {
"type": "string",
"pattern": "^[a-z2-7]{20}$"
},
"fqdn": {
"type": "string",
"maxLength": 253,
"pattern": "^[a-z2-7]{20}\\.(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\\.)+[a-z]{2,63}$"
},
"dns_host": {
"type": "string",
"maxLength": 253,
"pattern": "^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\\.)+[a-z]{2,63}$"
},
"acme_name": {
"type": "string",
"maxLength": 253,
"pattern": "^_acme-challenge\\.[a-z2-7]{20}\\.(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\\.)+[a-z]{2,63}$"
},
"node_id": {
"type": "string",
"pattern": "^[a-z0-9]{1,16}$"
},
"stream_id": {
"type": "string",
"pattern": "^[0-9a-f]{32}$"
},
"attach_secret": {
"type": "string",
"pattern": "^[A-Za-z0-9_-]{43}$"
},
"nonce": {
"type": "string",
"pattern": "^[A-Za-z0-9_-]{22,64}$"
},
"jti": {
"type": "string",
"pattern": "^[A-Za-z0-9_-]{16,128}$"
},
"kid": {
"type": "string",
"pattern": "^[A-Za-z0-9._-]{1,128}$"
},
"sha256_b64url": {
"type": "string",
"pattern": "^[A-Za-z0-9_-]{43}$"
},
"reservation_id": {
"type": "string",
"pattern": "^[0-9a-f]{32}$"
},
"ip": {
"type": "string",
"format": "ip"
},
"reason_word": {
"type": "string",
"pattern": "^[a-z_]{1,48}$"
},
"jwt": {
"type": "string",
"maxLength": 65552,
"pattern": "^[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+$"
},
"error_code": {
"type": "string",
"enum": [
"bad_hello",
"bad_command",
"unsupported_version",
"bad_proof",
"auth",
"clock_skew",
"unknown_stream",
"bad_attach",
"license_inactive",
"device_revoked",
"paused",
"busy",
"control_unavailable",
"line_too_long"
]
},
"close_reason": {
"type": "string",
"enum": [
"replaced",
"revoked",
"expired",
"released",
"killed",
"paused",
"drain",
"idle",
"protocol",
"shutdown"
]
},
"source_class": {
"type": "string",
"enum": [
"openai",
"hint",
"anon"
]
},
"authorize_status": {
"type": "string",
"enum": [
"active",
"gated",
"paused"
]
},
"budget_class": {
"type": "string",
"enum": [
"paid",
"trial",
"flex"
]
},
"public_jwk": {
"type": "object",
"required": [
"kty",
"crv",
"x"
],
"additionalProperties": false,
"properties": {
"kty": {
"const": "OKP"
},
"crv": {
"const": "Ed25519"
},
"x": {
"type": "string",
"pattern": "^[A-Za-z0-9_-]{43}$"
}
}
},
"node": {
"type": "object",
"required": [
"id",
"host",
"port"
],
"additionalProperties": false,
"properties": {
"id": {
"$ref": "#/$defs/node_id"
},
"host": {
"$ref": "#/$defs/dns_host"
},
"port": {
"type": "integer",
"minimum": 1,
"maximum": 65535
},
"pins": {
"type": "array",
"minItems": 1,
"maxItems": 4,
"uniqueItems": true,
"items": {
"$ref": "#/$defs/sha256_b64url"
}
}
}
},
"nodes": {
"type": "array",
"minItems": 1,
"maxItems": 8,
"items": {
"$ref": "#/$defs/node"
}
}
}
}
files/contracts/schemas/enroll-request.schema.json 12 lines · 600 B
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://all-mcp.fyi/contracts/v2/enroll-request.schema.json",
"title": "POST /v1/devices/e2e/enroll, device to control plane (all-control session plus DPoP, like the fleet heartbeat). Idempotent: install and migrate mint the label once; repair keeps it; reinstall mints a new one and tombstones the old.",
"type": "object",
"required": ["v", "intent"],
"additionalProperties": false,
"properties": {
"v": {"const": 1},
"intent": {"type": "string", "enum": ["install", "repair", "reinstall", "migrate"]}
}
}
files/contracts/schemas/enroll-response.schema.json 25 lines · 2.2 KB
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://all-mcp.fyi/contracts/v2/enroll-response.schema.json",
"title": "200 answer of enroll. host is stored per label (several zones can coexist); the device refuses an OK whose host differs from this one. enroll_after, when not null, is the earliest moment the device may start issuing and connecting (wave staging and certificate budget; for a computer that is not eligible it is a hold of policy.migration.ineligible_hold_s, so a device that ignores `eligible` still waits). cohort, eligible and reason (WP26, optional: a control plane older than the migration engine omits them) say which cohort the license is in, whether this computer may be moved to the encrypted path at all (Lifetime without updates and a legacy credential are never forced; a network that blocks 443 is the computer's own finding, reported as state compat_only, never a veto of the server) and, when it may not, one closed word. e2e_wave stays the global percentage (the computer is in a wave when its own bucket is below it); e2e_kill and copy_e2ee are global too.",
"type": "object",
"required": ["ok", "v", "device_id", "label", "host", "zone", "nodes", "enroll_after", "server_time", "e2e_wave", "e2e_kill", "copy_e2ee"],
"additionalProperties": false,
"properties": {
"ok": {"const": true},
"v": {"const": 1},
"device_id": {"$ref": "common.schema.json#/$defs/device_id"},
"label": {"$ref": "common.schema.json#/$defs/label"},
"host": {"$ref": "common.schema.json#/$defs/fqdn"},
"zone": {"$ref": "common.schema.json#/$defs/dns_host"},
"nodes": {"$ref": "common.schema.json#/$defs/nodes"},
"enroll_after": {"oneOf": [{"type": "null"}, {"$ref": "common.schema.json#/$defs/epoch_s"}]},
"server_time": {"$ref": "common.schema.json#/$defs/epoch_s"},
"e2e_wave": {"type": "integer", "minimum": 0, "maximum": 100},
"e2e_kill": {"type": "boolean"},
"copy_e2ee": {"type": "boolean"},
"cohort": {"type": "string", "enum": ["paid", "trial", "flex", "legacy"]},
"eligible": {"type": "boolean"},
"reason": {"type": "string", "enum": ["update_not_in_plan", "legacy_credential", "unsupported_version"]}
}
}
files/contracts/schemas/error-response.schema.json 15 lines · 748 B
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://all-mcp.fyi/contracts/v2/error-response.schema.json",
"title": "Every non-2xx answer of the end-to-end endpoints: a closed word, never text from the network, never a secret. status is only present next to license_inactive. retry_after_s accompanies 429 and 503.",
"type": "object",
"required": ["ok", "error"],
"additionalProperties": false,
"properties": {
"ok": {"const": false},
"error": {"type": "string", "pattern": "^[a-z_]{1,48}$"},
"status": {"type": "string", "pattern": "^[a-z_]{1,32}$"},
"retry_after_s": {"type": "integer", "minimum": 1, "maximum": 86400},
"enroll_after": {"$ref": "common.schema.json#/$defs/epoch_s"}
}
}
files/contracts/schemas/feed-response.schema.json 31 lines · 1.5 KB
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://all-mcp.fyi/contracts/v2/feed-response.schema.json",
"title": "GET /v1/tunnel/feed?since=<epoch>, node to control plane. Events carry a strictly increasing epoch; the answer holds every event with epoch greater than `since`, in order, each exactly once. reset=true means the node's cursor is older than the retained history: it must close every tunnel and re-authorize them.",
"type": "object",
"required": ["ok", "v", "epoch", "server_time", "events"],
"additionalProperties": false,
"properties": {
"ok": {"const": true},
"v": {"const": 1},
"epoch": {"$ref": "common.schema.json#/$defs/count"},
"server_time": {"$ref": "common.schema.json#/$defs/epoch_s"},
"reset": {"type": "boolean"},
"events": {
"type": "array",
"maxItems": 1000,
"items": {
"type": "object",
"required": ["epoch", "type", "device_id", "label", "at"],
"additionalProperties": false,
"properties": {
"epoch": {"type": "integer", "minimum": 1, "maximum": 9007199254740991},
"type": {"type": "string", "enum": ["revoke", "pause", "resume", "license_end", "release", "kill", "kill_clear"]},
"device_id": {"oneOf": [{"type": "null"}, {"$ref": "common.schema.json#/$defs/device_id"}]},
"label": {"oneOf": [{"type": "null"}, {"$ref": "common.schema.json#/$defs/label"}]},
"at": {"$ref": "common.schema.json#/$defs/epoch_s"}
}
}
}
}
}
files/contracts/schemas/fleet-e2e-block.schema.json 25 lines · 2.3 KB
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://all-mcp.fyi/contracts/v2/fleet-e2e-block.schema.json",
"title": "The `e2e` block of the fleet heartbeat (POST /v1/devices/heartbeat). The five base keys and the optional counters stay as they are (a control plane older than 2.0 refuses any other key with invalid_fleet_health_shape and the device falls back to the base shape); this object is one more optional key. Closed shape: no label, host, device id, address or free text. The Worker keeps only the latest block of each computer (overwritten) and counts over a 24-hour window from list metadata, with no identifier in any count (the contract, WP35 notes). bytes are the device's own totals for period_s. window is the device's random id for that byte window: a resend of an unaccepted window carries the same id and the same bytes, and the Worker adds a window's bytes to the day once (a 48 h marker per computer; a resend read before the first write has spread can still count twice, the contract, WP35 notes).",
"type": "object",
"required": ["v", "state", "cert_not_after", "nodes_connected", "nodes_configured", "last_error", "clock_skew", "ct_alert", "migrated", "cohort", "eligible", "window", "bytes_in", "bytes_out", "period_s"],
"additionalProperties": false,
"properties": {
"v": {"const": 1},
"state": {"type": "string", "enum": ["off", "unenrolled", "waiting", "issuing", "connecting", "ready", "degraded", "clock_skew", "compat_only", "killed"]},
"cert_not_after": {"oneOf": [{"type": "null"}, {"$ref": "common.schema.json#/$defs/epoch_s"}]},
"nodes_connected": {"type": "integer", "minimum": 0, "maximum": 8},
"nodes_configured": {"type": "integer", "minimum": 0, "maximum": 8},
"last_error": {"oneOf": [{"type": "null"}, {"$ref": "common.schema.json#/$defs/reason_word"}]},
"clock_skew": {"type": "boolean"},
"ct_alert": {"type": "boolean"},
"migrated": {"type": "boolean"},
"cohort": {"type": "string", "enum": ["paid", "trial", "flex", "legacy"]},
"eligible": {"type": "boolean"},
"window": {"type": "string", "pattern": "^[a-f0-9]{16}$"},
"bytes_in": {"$ref": "common.schema.json#/$defs/count"},
"bytes_out": {"$ref": "common.schema.json#/$defs/count"},
"period_s": {"type": "integer", "minimum": 1, "maximum": 604800}
}
}
files/contracts/schemas/jwks.schema.json 27 lines · 1.1 KB
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://all-mcp.fyi/contracts/v2/jwks.schema.json",
"title": "The control plane's published keys (GET /.well-known/device-jwks.json, 1.1 shape). The node pins the issuer, caches 300 s, reloads on an unknown kid, accepts at most 16 keys and fails closed after 30 minutes without a refresh (policy.jwks).",
"type": "object",
"required": ["keys"],
"additionalProperties": false,
"properties": {
"keys": {
"type": "array", "minItems": 1, "maxItems": 16,
"items": {
"type": "object",
"required": ["kty", "crv", "x", "kid", "alg", "use", "key_ops"],
"additionalProperties": false,
"properties": {
"kty": {"const": "OKP"},
"crv": {"const": "Ed25519"},
"x": {"type": "string", "pattern": "^[A-Za-z0-9_-]{43}$"},
"kid": {"$ref": "common.schema.json#/$defs/kid"},
"alg": {"const": "EdDSA"},
"use": {"const": "sig"},
"key_ops": {"type": "array", "minItems": 1, "maxItems": 1, "items": {"const": "verify"}}
}
}
}
}
}
files/contracts/schemas/node-assertion-claims.schema.json 23 lines · 1.4 KB
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://all-mcp.fyi/contracts/v2/node-assertion-claims.schema.json",
"title": "Claims of the node's assertion (header x-all-node-assertion, JWS typ node-assertion, kid of the node's own Ed25519 key). It authenticates one request of a node to /v1/tunnel/*: bound to the method, the URL and the SHA-256 of the exact body bytes, 60 s, one use (jti). bh of an empty body is the hash of the empty string. sub is the node id that the registered key belongs to. Unknown extra claims are ignored (a token is not a closed shape), the listed ones are not optional unless the schema says so.",
"type": "object",
"required": ["typ", "ver", "sub", "aud", "htm", "htu", "iat", "exp", "jti", "bh"],
"properties": {
"typ": {"const": "node-assertion"},
"ver": {"const": 1},
"sub": {"$ref": "common.schema.json#/$defs/node_id"},
"aud": {"const": "all-tunnel-control"},
"htm": {"type": "string", "enum": ["GET", "POST"]},
"htu": {
"type": "string",
"maxLength": 2048,
"pattern": "^https://[a-z0-9.-]{1,253}(?::[0-9]{1,5})?/v1/tunnel/[a-z]{1,32}(?:\\?[A-Za-z0-9=&_-]{0,64})?$"
},
"iat": {"$ref": "common.schema.json#/$defs/epoch_s"},
"exp": {"$ref": "common.schema.json#/$defs/epoch_s"},
"jti": {"$ref": "common.schema.json#/$defs/jti"},
"bh": {"$ref": "common.schema.json#/$defs/sha256_b64url"}
}
}
files/contracts/schemas/node-keys.schema.json 25 lines · 934 B
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://all-mcp.fyi/contracts/v2/node-keys.schema.json",
"title": "TUNNEL_NODE_KEYS of the control plane: the public Ed25519 key of each tunnel node. Revoking a node's key is removing its kid; the next request signed by it is refused.",
"type": "object",
"required": ["keys"],
"additionalProperties": false,
"properties": {
"keys": {
"type": "array", "minItems": 1, "maxItems": 16,
"items": {
"type": "object",
"required": ["kid", "node", "kty", "crv", "x"],
"additionalProperties": false,
"properties": {
"kid": {"$ref": "common.schema.json#/$defs/kid"},
"node": {"$ref": "common.schema.json#/$defs/node_id"},
"kty": {"const": "OKP"},
"crv": {"const": "Ed25519"},
"x": {"type": "string", "pattern": "^[A-Za-z0-9_-]{43}$"}
}
}
}
}
}
files/contracts/schemas/ok-body.schema.json 28 lines · 1.5 KB
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://all-mcp.fyi/contracts/v2/ok-body.schema.json",
"title": "Body of the node's OK line (answer to HELLO and to RENEW): what the control plane decided, relayed to the device. The windows of `policy` are the node's offer, capped at a day only to refuse nonsense: the device clamps each one to its own policy.json (WP13), it never trusts the offer beyond it.",
"type": "object",
"required": ["v", "node", "host", "status", "valid_until", "server_time", "policy"],
"additionalProperties": false,
"properties": {
"v": {"const": 2},
"node": {"$ref": "common.schema.json#/$defs/node_id"},
"host": {"$ref": "common.schema.json#/$defs/fqdn"},
"status": {"$ref": "common.schema.json#/$defs/authorize_status"},
"valid_until": {"$ref": "common.schema.json#/$defs/epoch_s"},
"server_time": {"$ref": "common.schema.json#/$defs/epoch_s"},
"policy": {
"type": "object",
"required": ["ping_interval_s", "dead_peer_s", "stream_idle_s", "stream_max_age_s", "renew_before_valid_until_s"],
"additionalProperties": false,
"properties": {
"ping_interval_s": {"type": "integer", "minimum": 1, "maximum": 86400},
"dead_peer_s": {"type": "integer", "minimum": 1, "maximum": 86400},
"stream_idle_s": {"type": "integer", "minimum": 1, "maximum": 86400},
"stream_max_age_s": {"type": "integer", "minimum": 1, "maximum": 86400},
"renew_before_valid_until_s": {"type": "integer", "minimum": 1, "maximum": 86400}
}
}
}
}
files/contracts/schemas/paywall.schema.json 38 lines · 2.0 KB
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://all-mcp.fyi/contracts/v2/paywall.schema.json",
"title": "The plans card inside a paywall statement: the shape the paywall module parse_paywall accepts (the device keeps its own field-by-field parser, which is stricter: distinct plan ids, byte limits, control characters).",
"type": "object",
"required": ["version", "status", "plan", "trial_ended_at", "upgrade_url", "plans", "note_es", "note_en", "text_es", "text_en"],
"additionalProperties": false,
"properties": {
"version": {"const": 1},
"status": {"const": "trial_ended"},
"plan": {"const": "trial"},
"trial_ended_at": {"$ref": "common.schema.json#/$defs/epoch_ms"},
"locale": {"type": "string", "enum": ["es", "en"]},
"upgrade_url": {"type": "string", "pattern": "^https://all-mcp\\.fyi/upgrade/(?:\\?lang=(?:es|en))?#[A-Za-z0-9._&=-]{8,400}$"},
"plans": {
"type": "array", "minItems": 3, "maxItems": 3,
"items": {
"type": "object",
"required": ["id", "label", "amount", "cadence", "updates", "computers"],
"additionalProperties": false,
"properties": {
"id": {"type": "string", "enum": ["monthly", "no_updates", "with_updates"]},
"label": {"type": "string", "minLength": 1, "maxLength": 64},
"amount": {"type": "string", "pattern": "^\\$[0-9]+\\.[0-9][0-9]$"},
"cadence": {"type": "string", "enum": ["monthly", "one_time"]},
"updates": {"type": "boolean"},
"computers": {"type": "integer", "minimum": 1, "maximum": 25}
}
}
},
"reactivation": {"type": "string", "minLength": 1, "maxLength": 200},
"is_error": {"type": "boolean"},
"note_es": {"type": "string", "minLength": 1, "maxLength": 2048},
"note_en": {"type": "string", "minLength": 1, "maxLength": 2048},
"text_es": {"type": "string", "minLength": 1, "maxLength": 2048},
"text_en": {"type": "string", "minLength": 1, "maxLength": 2048}
}
}
files/contracts/schemas/proof-claims.schema.json 17 lines · 1.1 KB
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://all-mcp.fyi/contracts/v2/proof-claims.schema.json",
"title": "Claims of the DPoP proof (header typ dpop+jwt carrying the device's public jwk) the device sends in PROOF and again, with a fresh nonce, after RENEW. htm is GET, htu is https://<node host>[:port if not 443]/all-tunnel/2, ath is base64url(SHA-256(session token)), nonce is the node's CHALLENGE. Unknown extra claims are ignored (a token is not a closed shape), the listed ones are not optional unless the schema says so.",
"type": "object",
"required": ["sub", "aud", "htm", "htu", "iat", "jti", "ath", "nonce"],
"properties": {
"sub": {"$ref": "common.schema.json#/$defs/device_id"},
"aud": {"const": "all-tunnel"},
"htm": {"const": "GET"},
"htu": {"type": "string", "maxLength": 300, "pattern": "^https://[a-z0-9.-]{1,253}(?::[0-9]{1,5})?/all-tunnel/2$"},
"iat": {"$ref": "common.schema.json#/$defs/epoch_s"},
"jti": {"$ref": "common.schema.json#/$defs/jti"},
"ath": {"$ref": "common.schema.json#/$defs/sha256_b64url"},
"nonce": {"$ref": "common.schema.json#/$defs/nonce"}
}
}
files/contracts/schemas/reconnect-state.schema.json 13 lines · 1.1 KB
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://all-mcp.fyi/contracts/v2/reconnect-state.schema.json",
"title": "The `reconnect` member of the 200 answer of GET /v1/devices/authorize (the relay path), WP21 reads it and WP26 writes it. Absent for a computer that must never see the card (Lifetime without updates, a legacy credential, a network that blocks 443, a computer that is not on 2.x, a computer whose encrypted path is not ready) and for every computer while the kill switch is on or the retirement criteria are not met. `eligible`: it can reach 2.x, its own self-check passed and it has not moved yet. `migrated`: it served a tool call on its encrypted address; `since` is that moment in epoch milliseconds. `locale` is the language the person signed up in (absent when unknown).",
"type": "object",
"required": ["state"],
"additionalProperties": false,
"properties": {
"state": {"type": "string", "enum": ["eligible", "migrated"]},
"since": {"type": "integer", "minimum": 1, "maximum": 4102444799999},
"locale": {"type": "string", "enum": ["es", "en"]}
}
}
files/contracts/schemas/selfcheck-result.schema.json 17 lines · 1.5 KB
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://all-mcp.fyi/contracts/v2/selfcheck-result.schema.json",
"title": "What a self-check says (WP26): somebody outside the computer's own process dialed the computer's public name (SNI = its host, TLS 1.3 only), verified the chain against the public roots, the name and the validity, compared the leaf's SubjectPublicKeyInfo hash with the one the computer expects, and asked GET /healthz. The tunnel node does it from its side (tunnel/selfcheck.go) and the computer does it from its own (the device's enrolment code); both give this shape. `ok` is true only when every step passed; `reason` is then null, otherwise the first step that failed (a closed word, never free text, never an address).",
"type": "object",
"required": ["v", "ok", "reason", "tls_version", "spki_sha256", "not_after", "healthz"],
"additionalProperties": false,
"properties": {
"v": {"const": 1},
"ok": {"type": "boolean"},
"reason": {"oneOf": [{"type": "null"}, {"type": "string", "enum": ["chain", "expired", "spki", "name", "tls_version", "healthz", "unreachable", "timeout"]}]},
"tls_version": {"oneOf": [{"type": "null"}, {"type": "string", "enum": ["1.2", "1.3"]}]},
"spki_sha256": {"oneOf": [{"type": "null"}, {"$ref": "common.schema.json#/$defs/sha256_b64url"}]},
"not_after": {"oneOf": [{"type": "null"}, {"$ref": "common.schema.json#/$defs/epoch_s"}]},
"healthz": {"type": "boolean"}
}
}
files/contracts/schemas/session-claims.schema.json 24 lines · 1.3 KB
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://all-mcp.fyi/contracts/v2/session-claims.schema.json",
"title": "Claims of a device session (typ device-session) as the control plane device-security module issues them (ver 2, 300 s). The tunnel node accepts only aud all-tunnel; the lifetime check exp - iat == policy.authorization.session_ttl_s is done in code. Unknown extra claims are ignored (a token is not a closed shape), the listed ones are not optional unless the schema says so.",
"type": "object",
"required": ["typ", "ver", "iss", "aud", "sub", "iat", "exp", "jti", "generation", "cnf"],
"properties": {
"typ": {"const": "device-session"},
"ver": {"const": 2},
"iss": {"const": "https://api.all-mcp.fyi"},
"aud": {"type": "string", "enum": ["all-control", "all-relay", "all-tunnel"]},
"sub": {"$ref": "common.schema.json#/$defs/device_id"},
"iat": {"$ref": "common.schema.json#/$defs/epoch_s"},
"exp": {"$ref": "common.schema.json#/$defs/epoch_s"},
"jti": {"$ref": "common.schema.json#/$defs/jti"},
"generation": {"type": "integer", "minimum": 1, "maximum": 9007199254740991},
"cnf": {
"type": "object",
"required": ["jwk"],
"additionalProperties": false,
"properties": {"jwk": {"$ref": "common.schema.json#/$defs/public_jwk"}}
}
}
}
files/contracts/schemas/statement-claims.schema.json 32 lines · 2.0 KB
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://all-mcp.fyi/contracts/v2/statement-claims.schema.json",
"title": "Claims of the control plane's signed access statement (typ device-access, aud all-device-access), as pushed by the node in STATEMENT and as returned next to authorize. Same keys and typ as 1.1 (ver stays 1: a 1.1 device ignores the added claims); seq is what makes an old statement unable to lift a pause or a paywall. Lifetime: mode active up to policy.authorization.access_ttl_paid_s (a verifier cannot tell a trial from a paid statement; the control plane issues access_ttl_trial_s for a trial and never past its end), mode paywall up to access_ttl_paywall_s. paused and resumed are never both present. paused.since and resumed.at are the license page's click moments in milliseconds (the 1.1 authorize shape). Unknown extra claims are ignored (a token is not a closed shape), the listed ones are not optional unless the schema says so.",
"type": "object",
"required": ["typ", "ver", "iss", "aud", "sub", "iat", "exp", "jti", "mode", "seq"],
"properties": {
"typ": {"const": "device-access"},
"ver": {"const": 1},
"iss": {"const": "https://api.all-mcp.fyi"},
"aud": {"const": "all-device-access"},
"sub": {"$ref": "common.schema.json#/$defs/device_id"},
"iat": {"$ref": "common.schema.json#/$defs/epoch_s"},
"exp": {"$ref": "common.schema.json#/$defs/epoch_s"},
"jti": {"$ref": "common.schema.json#/$defs/jti"},
"mode": {"type": "string", "enum": ["active", "paywall"]},
"seq": {"type": "integer", "minimum": 1, "maximum": 9007199254740991},
"paywall": {"$ref": "paywall.schema.json"},
"paused": {
"type": "object",
"required": ["since"],
"additionalProperties": false,
"properties": {"since": {"$ref": "common.schema.json#/$defs/epoch_ms"}}
},
"resumed": {
"type": "object",
"required": ["at"],
"additionalProperties": false,
"properties": {"at": {"$ref": "common.schema.json#/$defs/epoch_ms"}}
}
}
}
files/contracts/schemas/surfaces-file.schema.json 16 lines · 923 B
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://all-mcp.fyi/contracts/v2/surfaces-file.schema.json",
"title": "state_dir()/surfaces.json (mode 0600), written by the agent when the E2E surface listens. The terminator dials only host:port after checking that pid is the agent, and proves the listener holds `secret` (surface identity proof) before it sends the first ChatGPT byte. The secret is new at every start.",
"type": "object",
"required": ["v", "pid", "host", "port", "secret", "started_at"],
"additionalProperties": false,
"properties": {
"v": {"const": 1},
"pid": {"type": "integer", "minimum": 1, "maximum": 4294967295},
"host": {"const": "127.0.0.1"},
"port": {"type": "integer", "minimum": 1024, "maximum": 65535},
"secret": {"type": "string", "pattern": "^[A-Za-z0-9_-]{43}$"},
"started_at": {"$ref": "common.schema.json#/$defs/epoch_s"}
}
}
files/contracts/schemas/wire-v2.grammar.json 44 lines · 2.9 KB
{
"$comment": "Line grammar of tunnel protocol v2 (the contract section 3). A line is ASCII 0x20-0x7e, ends with LF (the LF counts toward the maximum), fields are separated by exactly one space, no field is empty. Verbs are upper case. `d2n` is device to node, `n2d` node to device. `max` names a policy key (bytes, LF included); lines without one use protocol.line_max_bytes. A field error is `bad_command` unless the field names its own `error`. `json` means the rest of the line after the verb and one space is one canonical JSON object (keys sorted, no whitespace, ASCII) validated against that schema.",
"version": 2,
"types": {
"version": {"pattern": "^[0-9]{1,3}$"},
"jwt": {"pattern": "^[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+$", "max_bytes_from": "protocol.jwt_max_bytes"},
"statement": {"pattern": "^[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+$"},
"nonce": {"pattern": "^[A-Za-z0-9_-]{22,64}$"},
"stream_id": {"pattern": "^[0-9a-f]{32}$"},
"attach_secret": {"pattern": "^[A-Za-z0-9_-]{43}$"},
"ip": {"format": "ip"},
"source_class": {"enum_from": "protocol.source_classes"},
"close_reason": {"enum_from": "protocol.close_reasons"},
"error_code": {"enum_from": "protocol.error_codes"},
"seconds": {"pattern": "^(?:0|[1-9][0-9]{0,5})$"}
},
"verbs": {
"HELLO": {"dir": "d2n", "fields": [
{"name": "version", "type": "version", "equals": "2", "error_if_number": "unsupported_version"},
{"name": "session", "type": "jwt", "error": "bad_hello"}
]},
"PROOF": {"dir": "d2n", "fields": [{"name": "proof", "type": "jwt", "error": "bad_proof"}]},
"RENEW": {"dir": "d2n", "fields": [{"name": "session", "type": "jwt", "error": "bad_hello"}]},
"PING": {"dir": "d2n", "fields": []},
"HINT": {"dir": "d2n", "repeat": {"name": "address", "type": "ip", "min": 1, "max_from": "protocol.hint_max_addresses"}},
"CLOSE": {"dir": "both", "optional": [{"name": "reason", "type": "close_reason"}]},
"ATTACH": {"dir": "d2n", "channel": "data", "fields": [
{"name": "stream_id", "type": "stream_id", "error": "bad_attach"},
{"name": "secret", "type": "attach_secret", "error": "bad_attach"}
]},
"CHALLENGE": {"dir": "n2d", "fields": [{"name": "nonce", "type": "nonce"}]},
"OK": {"dir": "n2d", "json": "ok-body.schema.json"},
"ERR": {"dir": "n2d", "fields": [{"name": "code", "type": "error_code"}]},
"OPEN": {"dir": "n2d", "max": "protocol.open_line_max_bytes", "fields": [
{"name": "stream_id", "type": "stream_id"},
{"name": "secret", "type": "attach_secret"},
{"name": "peer_ip", "type": "ip"},
{"name": "class", "type": "source_class"}
]},
"PONG": {"dir": "n2d", "fields": []},
"STATEMENT": {"dir": "n2d", "max": "protocol.statement_line_max_bytes", "fields": [{"name": "statement", "type": "statement"}]},
"DRAIN": {"dir": "n2d", "fields": [{"name": "quiet_s", "type": "seconds"}]}
}
}
files/contracts/ts/package.json 4 lines · 42 B
{
"private": true,
"type": "module"
}
files/contracts/ts/verify.ts 773 lines · 38.2 KB
// ALL 2.0 contracts, TypeScript reference verifier (the contract).
//
// The same checks as contracts/e2e_contracts.py and tunnel/contracts_verify_test.go, written independently: all three must give the same verdict
// AND the same error word on every golden vector of contracts/vectors/ (the Worker contract test runs this one). Spec and
// fixtures only: the control plane (WP02, WP03, WP04) writes its own handlers and tests them with these functions.
//
// It runs in Node (the test) and in a Worker: WebCrypto only (crypto.subtle), no node: imports, and it receives the policy, the schemas and
// the wire grammar as data (createContracts), so nothing here is typed twice. Erasable syntax only (no enums, no parameter properties), so
// Node runs the file as it is.
type Json = any;
type Dict = Record<string, Json>;
export type Verdict = { ok: boolean; error?: string; claims?: Dict; jti?: string };
export type Parsed = Dict & { verb: string };
export type ContractInput = { policy: Dict; schemas: Record<string, Json>; grammar: Dict };
export class ProtocolError extends Error {
code: string;
constructor(code: string) {
super(code);
this.code = code;
}
}
class Reject extends Error {
code: string;
constructor(code: string) {
super(code);
this.code = code;
}
}
const encoder = new TextEncoder();
/** Own keys only: `in` also finds what Object.prototype carries (constructor, toString, __proto__...), and the other verifiers have no such keys. */
const has = (value: object, key: string): boolean => Object.prototype.hasOwnProperty.call(value, key);
// ---------------------------------------------------------------------------------------------------------------------------- JSON
/** One JSON value: a repeated object key, a stray character after the value or an invalid escape is an error (JSON.parse accepts a repeated key). */
export function parseJson(text: string): Json {
let i = 0;
const fail = (): never => {
throw new Error("json");
};
const ws = () => {
while (i < text.length && " \t\n\r".includes(text[i])) i++;
};
const string = (): string => {
i++; // opening quote
let out = "";
for (;;) {
if (i >= text.length) fail();
const ch = text[i++];
if (ch === '"') return out;
if (ch.charCodeAt(0) < 0x20) fail();
if (ch !== "\\") {
out += ch;
continue;
}
const esc = text[i++];
const simple: Dict = { '"': '"', "\\": "\\", "/": "/", b: "\b", f: "\f", n: "\n", r: "\r", t: "\t" };
if (esc in simple) out += simple[esc];
else if (esc === "u") {
const hex = text.slice(i, i + 4);
if (!/^[0-9a-fA-F]{4}$/.test(hex)) fail();
out += String.fromCharCode(parseInt(hex, 16));
i += 4;
} else fail();
}
};
const value = (): Json => {
ws();
const ch = text[i];
if (ch === "{") {
i++;
const out: Dict = {};
ws();
if (text[i] === "}") {
i++;
return out;
}
for (;;) {
ws();
if (text[i] !== '"') fail();
const key = string();
if (Object.prototype.hasOwnProperty.call(out, key)) fail();
ws();
if (text[i++] !== ":") fail();
Object.defineProperty(out, key, { value: value(), enumerable: true, writable: true, configurable: true });
ws();
const next = text[i++];
if (next === "}") return out;
if (next !== ",") fail();
}
}
if (ch === "[") {
i++;
const out: Json[] = [];
ws();
if (text[i] === "]") {
i++;
return out;
}
for (;;) {
out.push(value());
ws();
const next = text[i++];
if (next === "]") return out;
if (next !== ",") fail();
}
}
if (ch === '"') return string();
for (const [word, literal] of [["true", true], ["false", false], ["null", null]] as const) {
if (text.startsWith(word, i)) {
i += word.length;
return literal;
}
}
const number = /^-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?/.exec(text.slice(i));
if (!number) return fail();
i += number[0].length;
return Number(number[0]);
};
const result = value();
ws();
if (i !== text.length) fail();
return result;
}
function noFloats(value: Json): boolean {
if (typeof value === "number") return Number.isInteger(value);
if (Array.isArray(value)) return value.every(noFloats);
if (value !== null && typeof value === "object") return Object.values(value).every(noFloats);
return true;
}
function isObject(value: Json): value is Dict {
return value !== null && typeof value === "object" && !Array.isArray(value);
}
/** The form of a JSON body on a wire line: keys sorted, no whitespace, ASCII only. */
export function canonicalJson(value: Json): string {
if (value === null) return "null";
if (typeof value === "boolean") return value ? "true" : "false";
if (typeof value === "number") return String(value);
if (typeof value === "string") {
let out = '"';
for (let k = 0; k < value.length; k++) {
const code = value.charCodeAt(k);
const ch = value[k];
if (ch === '"') out += '\\"';
else if (ch === "\\") out += "\\\\";
else if (ch === "\n") out += "\\n";
else if (ch === "\r") out += "\\r";
else if (ch === "\t") out += "\\t";
else if (ch === "\b") out += "\\b";
else if (ch === "\f") out += "\\f";
else if (code < 0x20 || code > 0x7e) out += "\\u" + code.toString(16).padStart(4, "0");
else out += ch;
}
return out + '"';
}
if (Array.isArray(value)) return "[" + value.map(canonicalJson).join(",") + "]";
return "{" + Object.keys(value).sort().map((key) => canonicalJson(key) + ":" + canonicalJson(value[key])).join(",") + "}";
}
function equal(a: Json, b: Json): boolean {
if (a === null || b === null) return a === b;
if (typeof a !== typeof b) return false;
if (Array.isArray(a)) return Array.isArray(b) && a.length === b.length && a.every((item, k) => equal(item, b[k]));
if (typeof a === "object") {
const keys = Object.keys(a);
return !Array.isArray(b) && keys.length === Object.keys(b).length && keys.every((key) => key in b && equal(a[key], b[key]));
}
return a === b;
}
// ---------------------------------------------------------------------------------------------------------------------------- bytes, hashes
function b64uDecode(text: string): Uint8Array {
if (typeof text !== "string" || !/^[A-Za-z0-9_-]+$/.test(text)) throw new Error("base64url");
const raw = atob(text.replace(/-/g, "+").replace(/_/g, "/") + "=".repeat((4 - (text.length % 4)) % 4));
const bytes = Uint8Array.from(raw, (c) => c.charCodeAt(0));
if (b64uEncode(bytes) !== text) throw new Error("base64url");
return bytes;
}
function b64uEncode(bytes: Uint8Array): string {
let binary = "";
for (const byte of bytes) binary += String.fromCharCode(byte);
return btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/g, "");
}
async function sha256B64(data: Uint8Array | string): Promise<string> {
return b64uEncode(new Uint8Array(await crypto.subtle.digest("SHA-256", typeof data === "string" ? encoder.encode(data) : data)));
}
async function hmacB64(key: Uint8Array, message: string): Promise<string> {
const imported = await crypto.subtle.importKey("raw", key, { name: "HMAC", hash: "SHA-256" }, false, ["sign"]);
return b64uEncode(new Uint8Array(await crypto.subtle.sign("HMAC", imported, encoder.encode(message))));
}
function sameText(a: string, b: string): boolean {
if (a.length !== b.length) return false;
let diff = 0;
for (let k = 0; k < a.length; k++) diff |= a.charCodeAt(k) ^ b.charCodeAt(k);
return diff === 0;
}
async function ed25519Ok(x: string, signature: Uint8Array, input: string): Promise<boolean> {
if (signature.length !== 64) return false;
try {
const key = await crypto.subtle.importKey("jwk", { kty: "OKP", crv: "Ed25519", x }, "Ed25519", false, ["verify"]);
return await crypto.subtle.verify("Ed25519", key, signature, encoder.encode(input));
} catch {
return false;
}
}
// ---------------------------------------------------------------------------------------------------------------------------- addresses
const V4 = /^(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])(?:\.(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])){3}$/;
/** `format: ip`: a textual IPv4 or IPv6 address, lower case, no zone, no brackets, no spaces (RFC 4291 text forms, dotted-quad tail allowed). */
export function validIp(value: Json): boolean {
if (typeof value !== "string" || value === "" || value !== value.toLowerCase() || /[%[\] \t\r\n/]/.test(value)) return false;
if (!value.includes(":")) return V4.test(value);
const halves = value.split("::");
if (halves.length > 2) return false;
const groups = (part: string): string[] => (part === "" ? [] : part.split(":"));
const all = halves.length === 2 ? [...groups(halves[0]), ...groups(halves[1])] : groups(halves[0]);
let count = 0;
for (let k = 0; k < all.length; k++) {
const group = all[k];
// a dotted-quad tail must end the whole address: "1.2.3.4::" is not an address
if (k === all.length - 1 && group.includes(".") && (halves.length === 1 || halves[1] !== "")) {
if (!V4.test(group)) return false;
count += 2;
} else {
if (!/^[0-9a-f]{1,4}$/.test(group)) return false;
count += 1;
}
}
return halves.length === 2 ? count <= 7 : count === 8;
}
// ---------------------------------------------------------------------------------------------------------------------------- the contracts
export function createContracts(input: ContractInput) {
const { policy, schemas, grammar } = input;
const pol = (dotted: string): Json => dotted.split(".").reduce((node: Json, part) => node[part], policy);
const anchored = (pattern: string): RegExp => {
if (!pattern.startsWith("^") || !pattern.endsWith("$")) throw new Error("contract patterns are anchored: " + pattern);
return new RegExp(pattern);
};
const labelRe = anchored(pol("identity.label_regex"));
const fqdnRe = anchored(pol("identity.fqdn_regex"));
const nonceRe = anchored(pol("identity.nonce_regex"));
const kidRe = anchored(pol("identity.kid_regex"));
const patterns = new Map<string, RegExp>();
const pattern = (text: string): RegExp => {
if (!patterns.has(text)) patterns.set(text, anchored(text));
return patterns.get(text) as RegExp;
};
const labelOk = (value: Json): boolean => typeof value === "string" && labelRe.test(value);
const fqdnOk = (value: Json, zones?: string[]): boolean => {
if (typeof value !== "string" || value.length > pol("identity.fqdn_max_length") || !fqdnRe.test(value)) return false;
return zones === undefined || zones.includes(value.slice(value.indexOf(".") + 1));
};
/** The detector a node or a device builds for its own address in a zone: the policy's label pattern, one dot, the zone. */
const hostPattern = (zone: string): RegExp => {
const label: string = pol("identity.label_regex");
return new RegExp("^" + label.slice(1, -1) + "\\." + zone.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") + "$");
};
// ---- JSON Schema subset (the keywords the schemas use; the same list in the Python and Go validators)
const typeOk = (kind: string, value: Json): boolean => {
switch (kind) {
case "string": return typeof value === "string";
case "boolean": return typeof value === "boolean";
case "null": return value === null;
case "object": return isObject(value);
case "array": return Array.isArray(value);
case "integer": return typeof value === "number" && Number.isInteger(value);
case "number": return typeof value === "number";
default: return false;
}
};
const resolve = (doc: string, ref: string): [string, Json] => {
const [file, pointer] = ref.split("#");
const target = file || doc;
let node: Json = schemas[target];
if (pointer) for (const part of pointer.replace(/^\//, "").split("/")) node = node[part.replace(/~1/g, "/").replace(/~0/g, "~")];
return [target, node];
};
const check = (doc: string, schema: Dict, value: Json, path: string, errors: string[]): void => {
const at = path || "/";
if ("$ref" in schema) {
const [target, node] = resolve(doc, schema.$ref);
check(target, node, value, path, errors);
}
if ("const" in schema && !equal(value, schema.const)) errors.push(`${at}: const`);
if ("enum" in schema && !schema.enum.some((item: Json) => equal(value, item))) errors.push(`${at}: enum`);
if ("type" in schema) {
const kinds: string[] = Array.isArray(schema.type) ? schema.type : [schema.type];
if (!kinds.some((kind) => typeOk(kind, value))) {
errors.push(`${at}: type`);
return;
}
}
if ("oneOf" in schema) {
let hits = 0;
for (const option of schema.oneOf) {
const trial: string[] = [];
check(doc, option, value, path, trial);
if (trial.length === 0) hits++;
}
if (hits !== 1) errors.push(`${at}: oneOf`);
}
if (typeof value === "string") {
const length = [...value].length;
if ("minLength" in schema && length < schema.minLength) errors.push(`${at}: minLength`);
if ("maxLength" in schema && length > schema.maxLength) errors.push(`${at}: maxLength`);
if ("pattern" in schema && !pattern(schema.pattern).test(value)) errors.push(`${at}: pattern`);
if (schema.format === "ip" && !validIp(value)) errors.push(`${at}: format`);
}
if (typeof value === "number") {
if ("minimum" in schema && value < schema.minimum) errors.push(`${at}: minimum`);
if ("maximum" in schema && value > schema.maximum) errors.push(`${at}: maximum`);
}
if (isObject(value)) {
for (const key of schema.required ?? []) if (!has(value, key)) errors.push(`${path}/${key}: required`);
const props: Dict = schema.properties ?? {};
for (const key of Object.keys(value).sort()) {
if (has(props, key)) check(doc, props[key], value[key], `${path}/${key}`, errors);
else if (schema.additionalProperties === false) errors.push(`${path}/${key}: additionalProperties`);
}
}
if (Array.isArray(value)) {
if ("minItems" in schema && value.length < schema.minItems) errors.push(`${at}: minItems`);
if ("maxItems" in schema && value.length > schema.maxItems) errors.push(`${at}: maxItems`);
if (schema.uniqueItems && value.some((a, i) => value.some((b, j) => j > i && equal(a, b)))) errors.push(`${at}: uniqueItems`);
if ("items" in schema) value.forEach((item, k) => check(doc, schema.items, item, `${path}/${k}`, errors));
}
};
const validate = (name: string, instance: Json): string[] => {
const errors: string[] = [];
check(name, schemas[name], instance, "", errors);
return errors;
};
// ---- the wire grammar
const fieldOk = (kind: string, token: string): boolean => {
const spec: Dict = grammar.types[kind];
if (spec.max_bytes_from && token.length > pol(spec.max_bytes_from)) return false;
if (spec.pattern && !pattern(spec.pattern).test(token)) return false;
if (spec.format === "ip" && !validIp(token)) return false;
if (spec.enum_from && !pol(spec.enum_from).includes(token)) return false;
return true;
};
/** Parse one protocol line (the text before the LF) travelling in `direction` ("d2n" or "n2d"). Throws ProtocolError. */
const parseLine = (line: string, direction: string): Parsed => {
if (typeof line !== "string" || [...line].some((ch) => ch.charCodeAt(0) < 0x20 || ch.charCodeAt(0) > 0x7e)) throw new ProtocolError("bad_command");
const cut = line.indexOf(" ");
const verb = cut < 0 ? line : line.slice(0, cut);
const hasSpace = cut >= 0;
const rest = hasSpace ? line.slice(cut + 1) : "";
const spec: Dict | undefined = Object.prototype.hasOwnProperty.call(grammar.verbs, verb) ? grammar.verbs[verb] : undefined;
const limit: number = spec && spec.max ? pol(spec.max) : pol("protocol.line_max_bytes");
if (line.length + 1 > limit) throw new ProtocolError("line_too_long");
if (!spec || (spec.dir !== direction && spec.dir !== "both") || line === "" || (hasSpace && rest === "")) throw new ProtocolError("bad_command");
const out: Parsed = { verb };
if (spec.json) {
if (!hasSpace) throw new ProtocolError("bad_command");
let body: Json;
try {
body = parseJson(rest);
} catch {
throw new ProtocolError("bad_command");
}
if (!noFloats(body) || validate(spec.json, body).length > 0) throw new ProtocolError("bad_command");
out.body = body;
return out;
}
const tokens = hasSpace ? rest.split(" ") : [];
if (tokens.some((token) => token === "")) throw new ProtocolError("bad_command");
if (spec.repeat) {
const rule = spec.repeat;
if (tokens.length < rule.min || tokens.length > pol(rule.max_from) || !tokens.every((token) => fieldOk(rule.type, token))) throw new ProtocolError("bad_command");
out.addresses = tokens;
return out;
}
const fields: Dict[] = spec.fields ?? [];
const optional: Dict[] = spec.optional ?? [];
if (tokens.length < fields.length || tokens.length > fields.length + optional.length) throw new ProtocolError("bad_command");
const rules = [...fields, ...optional];
tokens.forEach((token, k) => {
const rule = rules[k];
if ("equals" in rule && token !== rule.equals) throw new ProtocolError(fieldOk(rule.type, token) ? rule.error_if_number : "bad_command");
if (!fieldOk(rule.type, token)) throw new ProtocolError(rule.error ?? "bad_command");
out[rule.name] = token;
});
return out;
};
/** The canonical line of a parsed message (the inverse of parseLine, which it is checked against). */
const formatLine = (message: Parsed): string => {
const spec: Dict = grammar.verbs[message.verb];
let line: string;
if (spec.json) line = `${message.verb} ${canonicalJson(message.body)}`;
else if (spec.repeat) line = [message.verb, ...message.addresses].join(" ");
else {
const rules: Dict[] = [...(spec.fields ?? []), ...(spec.optional ?? [])];
line = [message.verb, ...rules.filter((rule) => has(message, rule.name)).map((rule) => message[rule.name])].join(" ");
}
parseLine(line, spec.dir === "both" ? "d2n" : spec.dir);
return line;
};
// ---- tokens
const fail = (error: string): Verdict => ({ ok: false, error });
const jwsParts = (token: Json, maxBytes: number) => {
try {
if (typeof token !== "string" || [...token].some((ch) => ch.charCodeAt(0) > 0x7f) || token.length > maxBytes) throw new Error();
const parts = token.split(".");
if (parts.length !== 3) throw new Error();
// ignoreBOM keeps a leading U+FEFF in the text, so parseJson refuses it (Python and Go refuse it too; the default decoder drops it)
const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
const header = parseJson(decoder.decode(b64uDecode(parts[0])));
const claims = parseJson(decoder.decode(b64uDecode(parts[1])));
const signature = parts[2] === "" ? new Uint8Array(0) : b64uDecode(parts[2]);
if (!isObject(header) || !isObject(claims)) throw new Error();
if (["crit", "b64", "jku", "x5u", "x5c"].some((name) => has(header, name))) throw new Error();
if (!noFloats(claims)) throw new Error();
return { header, claims, signature, input: `${parts[0]}.${parts[1]}` };
} catch {
throw new Reject("malformed_token");
}
};
const jwkX = (value: Json): string => {
if (!isObject(value) || value.kty !== "OKP" || value.crv !== "Ed25519" || has(value, "d") || typeof value.x !== "string" || value.x.length !== 43) {
throw new Reject("malformed_token");
}
try {
if (b64uDecode(value.x).length !== 32) throw new Error();
} catch {
throw new Reject("malformed_token");
}
return value.x;
};
const claimsOk = (claims: Dict, schema: string, ...ignore: string[]): boolean =>
validate(schema, claims).every((error) => ignore.some((name) => error.startsWith(`/${name}:`)));
const parseJwks = (doc: Json): Map<string, string> => {
if (validate("jwks.schema.json", doc).length > 0) throw new Reject("bad_jwks");
const keys = new Map<string, string>();
for (const key of doc.keys) {
if (keys.has(key.kid)) throw new Reject("bad_jwks");
keys.set(key.kid, key.x);
}
return keys;
};
const parseNodeKeys = (doc: Json): Map<string, { node: string; x: string }> => {
if (validate("node-keys.schema.json", doc).length > 0) throw new Reject("bad_node_keys");
const keys = new Map<string, { node: string; x: string }>();
for (const key of doc.keys) {
if (keys.has(key.kid)) throw new Reject("bad_node_keys");
keys.set(key.kid, { node: key.node, x: key.x });
}
return keys;
};
const signedByJwks = async (token: Json, keys: Map<string, string>, typ: string) => {
const jws = jwsParts(token, pol("protocol.statement_line_max_bytes"));
if (jws.header.alg !== "EdDSA") throw new Reject("bad_alg");
const kid = jws.header.kid;
if (typeof kid !== "string" || !kidRe.test(kid) || !keys.has(kid)) throw new Reject("unknown_kid");
if (!(await ed25519Ok(keys.get(kid) as string, jws.signature, jws.input))) throw new Reject("bad_signature");
if (jws.header.typ !== typ || jws.claims.typ !== typ) throw new Reject("wrong_type");
return jws;
};
const run = async (body: () => Promise<Verdict>): Promise<Verdict> => {
try {
return await body();
} catch (error) {
if (error instanceof Reject) return fail(error.code);
throw error;
}
};
/** A device session as the node accepts it at HELLO and RENEW. Order of failure: malformed_token, bad_alg, unknown_kid, bad_signature, wrong_type,
* wrong_issuer, wrong_audience, bad_claims, bad_lifetime, not_yet_valid, expired. */
const verifySession = (token: Json, o: { now: number; audience: string; keys: Map<string, string>; skew: number }): Promise<Verdict> => run(async () => {
if (typeof token !== "string" || token.length > pol("protocol.jwt_max_bytes")) throw new Reject("malformed_token");
const { claims } = await signedByJwks(token, o.keys, "device-session");
if (claims.ver !== 2) throw new Reject("wrong_type");
if (claims.iss !== pol("identity.issuer")) throw new Reject("wrong_issuer");
if (claims.aud !== o.audience) throw new Reject("wrong_audience");
if (!claimsOk(claims, "session-claims.schema.json", "aud")) throw new Reject("bad_claims");
if (claims.exp - claims.iat !== pol("authorization.session_ttl_s")) throw new Reject("bad_lifetime");
if (claims.iat > o.now + o.skew) throw new Reject("not_yet_valid");
if (claims.exp <= o.now) throw new Reject("expired");
return { ok: true, claims, jti: claims.jti };
});
/** The DPoP proof of PROOF. Order of failure: malformed_token, bad_alg, wrong_type, wrong_key, bad_signature, wrong_subject, wrong_audience,
* wrong_htm, wrong_htu, wrong_ath, wrong_nonce, bad_claims, not_yet_valid, stale_proof, replayed_jti. */
const verifyProof = (proof: Json, o: { now: number; sessionToken: string; sessionClaims: Dict; nonce: string; htu: string; skew: number; seen: Set<string> }): Promise<Verdict> => run(async () => {
if (typeof proof !== "string" || proof.length > pol("protocol.jwt_max_bytes")) throw new Reject("malformed_token");
const jws = jwsParts(proof, pol("protocol.jwt_max_bytes"));
if (jws.header.alg !== "EdDSA") throw new Reject("bad_alg");
if (jws.header.typ !== "dpop+jwt") throw new Reject("wrong_type");
const x = jwkX(jws.header.jwk);
if (x !== o.sessionClaims.cnf.jwk.x) throw new Reject("wrong_key");
if (!(await ed25519Ok(x, jws.signature, jws.input))) throw new Reject("bad_signature");
const claims = jws.claims;
if (claims.sub !== o.sessionClaims.sub) throw new Reject("wrong_subject");
if (claims.aud !== pol("identity.aud_tunnel")) throw new Reject("wrong_audience");
if (claims.htm !== pol("protocol.proof_htm")) throw new Reject("wrong_htm");
if (claims.htu !== o.htu) throw new Reject("wrong_htu");
if (claims.ath !== (await sha256B64(o.sessionToken))) throw new Reject("wrong_ath");
if (claims.nonce !== o.nonce) throw new Reject("wrong_nonce");
if (!claimsOk(claims, "proof-claims.schema.json", "htu")) throw new Reject("bad_claims");
if (claims.iat > o.now + o.skew) throw new Reject("not_yet_valid");
if (claims.iat + pol("authorization.proof_ttl_s") <= o.now) throw new Reject("stale_proof");
if (o.seen.has(claims.jti)) throw new Reject("replayed_jti");
return { ok: true, claims, jti: claims.jti };
});
/** A node's request authentication (header x-all-node-assertion). Order of failure: malformed_token, bad_alg, unknown_kid, bad_signature,
* wrong_type, wrong_audience, wrong_subject, wrong_htm, wrong_htu, bad_claims, bad_lifetime, not_yet_valid, expired, wrong_body_hash, replayed_jti. */
const verifyNodeAssertion = (token: Json, o: { now: number; keys: Map<string, { node: string; x: string }>; method: string; url: string; body: Uint8Array; skew: number; seen: Set<string> }): Promise<Verdict> => run(async () => {
const jws = jwsParts(token, pol("protocol.jwt_max_bytes"));
if (jws.header.alg !== "EdDSA") throw new Reject("bad_alg");
const kid = jws.header.kid;
if (typeof kid !== "string" || !kidRe.test(kid) || !o.keys.has(kid)) throw new Reject("unknown_kid");
const key = o.keys.get(kid) as { node: string; x: string };
if (!(await ed25519Ok(key.x, jws.signature, jws.input))) throw new Reject("bad_signature");
const claims = jws.claims;
if (jws.header.typ !== "node-assertion" || claims.typ !== "node-assertion" || claims.ver !== 1) throw new Reject("wrong_type");
if (claims.aud !== pol("identity.aud_node_assertion")) throw new Reject("wrong_audience");
if (claims.sub !== key.node) throw new Reject("wrong_subject");
if (claims.htm !== o.method) throw new Reject("wrong_htm");
if (claims.htu !== o.url) throw new Reject("wrong_htu");
if (!claimsOk(claims, "node-assertion-claims.schema.json", "htm", "htu", "aud", "sub")) throw new Reject("bad_claims");
if (claims.exp - claims.iat !== pol("authorization.node_assertion_ttl_s")) throw new Reject("bad_lifetime");
if (claims.iat > o.now + o.skew) throw new Reject("not_yet_valid");
if (claims.exp <= o.now) throw new Reject("expired");
if (claims.bh !== (await sha256B64(o.body))) throw new Reject("wrong_body_hash");
if (o.seen.has(claims.jti)) throw new Reject("replayed_jti");
return { ok: true, claims, jti: claims.jti };
});
/** The control plane's signed access statement as the device accepts it. Order of failure: malformed_token, bad_alg, unknown_kid, bad_signature,
* wrong_type, wrong_issuer, wrong_audience, wrong_subject, bad_claims, bad_lifetime, not_yet_valid, stale_seq. An expired statement still verifies. */
const verifyStatement = (token: Json, o: { now: number; keys: Map<string, string>; deviceId: string; lastSeq: number }): Promise<Verdict> => run(async () => {
const { claims } = await signedByJwks(token, o.keys, "device-access");
if (claims.ver !== 1) throw new Reject("wrong_type");
if (claims.iss !== pol("identity.issuer")) throw new Reject("wrong_issuer");
if (claims.aud !== pol("identity.aud_access")) throw new Reject("wrong_audience");
if (claims.sub !== o.deviceId) throw new Reject("wrong_subject");
if (!claimsOk(claims, "statement-claims.schema.json", "aud", "sub") || has(claims, "paywall") !== (claims.mode === "paywall") ||
(has(claims, "paused") && has(claims, "resumed"))) throw new Reject("bad_claims");
const ttl = claims.exp - claims.iat;
if (!(ttl > 0 && ttl <= pol(claims.mode === "paywall" ? "authorization.access_ttl_paywall_s" : "authorization.access_ttl_paid_s"))) throw new Reject("bad_lifetime");
if (claims.iat > o.now + pol("authorization.access_future_slack_s")) throw new Reject("not_yet_valid");
if (claims.seq < o.lastSeq) throw new Reject("stale_seq");
return { ok: true, claims, jti: claims.jti };
});
/** Schema plus the rules a schema cannot say: events strictly increasing and after `since`, epoch not behind the last event, kill and kill_clear
* carry no device, every other event carries both device and label. */
const checkFeed = (body: Json, since: number): string[] => {
const errors = validate("feed-response.schema.json", body);
if (errors.length > 0) return errors;
let previous = since;
body.events.forEach((event: Dict, k: number) => {
if (event.epoch <= previous) errors.push(`/events/${k}: order`);
previous = Math.max(previous, event.epoch);
const global = event.type === "kill" || event.type === "kill_clear";
if (global !== (event.device_id === null && event.label === null)) errors.push(`/events/${k}: scope`);
if ((event.device_id === null) !== (event.label === null)) errors.push(`/events/${k}: scope`);
});
if (body.epoch < previous) errors.push("/epoch: behind events");
return errors;
};
// ---- surface identity proof
const surfaceKey = (secret: string): Uint8Array => {
const raw = b64uDecode(secret);
if (raw.length !== 32) throw new Error("secret");
return raw;
};
const surfaceMac = (secret: string, nonce: string): Promise<string> => hmacB64(surfaceKey(secret), "all-surface-challenge-v1\n" + nonce);
const surfaceProof = (secret: string, nonce: string, pid: number, port: number): Promise<string> =>
hmacB64(surfaceKey(secret), `all-surface-proof-v1\n${nonce}\n${pid}\n${port}`);
const verifySurfaceChallenge = async (secret: string, nonce: string, mac: Json): Promise<Verdict> => {
try {
if (typeof nonce !== "string" || !nonceRe.test(nonce) || typeof mac !== "string") return fail("bad_challenge");
return sameText(await surfaceMac(secret, nonce), mac) ? { ok: true } : fail("bad_challenge");
} catch {
return fail("bad_challenge");
}
};
const verifySurfaceProof = async (secret: string, nonce: string, pid: number, port: number, proof: Json): Promise<Verdict> => {
try {
if (typeof nonce !== "string" || !nonceRe.test(nonce) || typeof proof !== "string") return fail("bad_proof");
return sameText(await surfaceProof(secret, nonce, pid, port), proof) ? { ok: true } : fail("bad_proof");
} catch {
return fail("bad_proof");
}
};
const identityRequest = (host: string, port: number, nonce: string, mac: string): string =>
`GET ${pol("surface.identity_path")} HTTP/1.1\r\nHost: ${host}:${port}\r\n${pol("surface.identity_challenge_header")}: ${nonce}\r\n` +
`${pol("surface.identity_mac_header")}: ${mac}\r\nConnection: keep-alive\r\n\r\n`;
// ---- node behaviour
/** Does the recorded behaviour of ONE node obey the contract? Returns "" when it does, otherwise the first violation word. */
const checkTranscript = async (doc: Dict): Promise<string> => {
const ctx = doc.ctx;
const keys = parseJwks(ctx.jwks);
const skew: number = ctx.skew;
type Conn = { phase: string; session: string; claims: Dict | null; sessionOk: boolean; kind: string; nonce: string; proof: string; authorized: boolean;
reply: Dict | null; host: string; status: string; validUntil: number; superseded: boolean; sub: string };
const conns = new Map<string, Conn>();
const seen = new Set<string>();
const streams = new Map<string, { secret: string; t: number; consumed: boolean }>();
const secretsUsed = new Set<string>();
const hellos = new Map<string, { ip: string; cls: string }[]>(); // per host: the peer_ip and class of each complete ClientHello still waiting for its OPEN
let ended = false;
let now = 0;
for (const event of doc.events as Dict[]) {
now = event.t;
const kind: string = event.ev;
let conn: Conn | undefined;
if (typeof event.conn === "string" && ["recv", "send", "cp", "cp_reply"].includes(kind)) {
if (!conns.has(event.conn)) {
conns.set(event.conn, { phase: "new", session: "", claims: null, sessionOk: false, kind: "hello", nonce: "", proof: "none", authorized: false,
reply: null, host: "", status: "", validUntil: 0, superseded: false, sub: "" });
}
conn = conns.get(event.conn);
}
if (kind === "client_hello") {
if (event.complete === true) hellos.set(event.sni, [...(hellos.get(event.sni) ?? []), { ip: event.peer_ip, cls: event.class }]);
} else if (kind === "recv" && conn) {
let message: Parsed;
try {
message = parseLine(event.line, "d2n");
} catch {
continue;
}
const verb = message.verb;
if ((verb === "HELLO" && conn.phase === "new") || (verb === "RENEW" && conn.phase === "ready")) {
const result = await verifySession(message.session, { now, audience: pol("identity.aud_tunnel"), keys, skew });
conn.sessionOk = result.ok && (verb === "HELLO" || (result.claims as Dict).sub === conn.sub);
conn.session = message.session;
conn.claims = result.ok ? (result.claims as Dict) : null;
conn.kind = verb === "HELLO" ? "hello" : "renew";
conn.nonce = "";
conn.proof = "none";
conn.authorized = false;
conn.reply = null;
if (verb === "HELLO") conn.phase = "hello";
} else if (verb === "PROOF" && conn.sessionOk && conn.nonce !== "") {
const result = await verifyProof(message.proof, { now, sessionToken: conn.session, sessionClaims: conn.claims as Dict, nonce: conn.nonce,
htu: ctx.node_url, skew, seen });
conn.proof = result.ok ? "ok" : result.error === "replayed_jti" ? "replayed" : "invalid";
conn.authorized = false;
conn.reply = null;
if (result.ok && result.jti !== undefined) seen.add(result.jti);
}
} else if (kind === "cp" && conn) {
const body = event.body;
if (validate("authorize-request.schema.json", body).length > 0 || conn.claims === null || body.device_id !== conn.claims.sub ||
body.session_jti !== conn.claims.jti || body.reason !== conn.kind || body.node !== ctx.node_id) return "bad_authorize_request";
if (conn.proof === "ok") conn.authorized = true;
} else if (kind === "cp_reply" && conn) {
conn.reply = conn.authorized ? event : null;
} else if (kind === "send" && conn) {
let message: Parsed;
try {
message = parseLine(event.line, "n2d");
} catch {
return "bad_line";
}
if (message.verb === "CHALLENGE") {
if (!conn.sessionOk || (conn.phase !== "hello" && conn.phase !== "ready")) return "challenge_for_invalid_session";
conn.nonce = message.nonce;
} else if (message.verb === "OK") {
if (event.line !== "OK " + canonicalJson(message.body)) return "bad_line";
if (conn.proof === "none") return "ok_without_proof";
if (conn.proof === "replayed") return "replayed_proof_accepted";
if (conn.proof !== "ok") return "ok_with_invalid_proof";
if (!conn.authorized || conn.reply === null || conn.reply.status !== 200) return conn.kind === "renew" ? "renew_without_authorize" : "ok_without_authorize";
const body = message.body;
const answer = conn.reply.body;
if (body.host !== answer.host) return "host_not_from_control_plane";
if (body.status !== answer.status || body.valid_until !== answer.valid_until || !equal(body.policy, answer.policy) || body.node !== ctx.node_id) {
return "ok_mismatch_control_plane";
}
conn.phase = "ready";
conn.host = body.host;
conn.status = body.status;
conn.validUntil = body.valid_until;
conn.sub = (conn.claims as Dict).sub;
conn.proof = "none";
conn.authorized = false;
conn.reply = null;
for (const other of conns.values()) if (other !== conn && other.phase === "ready" && other.sub === conn.sub) other.superseded = true;
} else if (message.verb === "OPEN") {
if (conn.phase !== "ready" || conn.status === "") return "open_before_ok";
if (conn.status === "paused") return "open_when_paused";
if (now >= conn.validUntil) return "open_after_valid_until";
const pending = hellos.get(conn.host) ?? [];
if (pending.length === 0) return [...hellos.values()].some((list) => list.length > 0) ? "open_wrong_host" : "open_without_client_hello";
const waiting = pending.findIndex((hello) => hello.ip === message.peer_ip && hello.cls === message.class);
if (waiting < 0) return "open_wrong_client";
if (streams.has(message.stream_id) || secretsUsed.has(message.secret)) return "duplicate_stream";
pending.splice(waiting, 1);
streams.set(message.stream_id, { secret: message.secret, t: now, consumed: false });
secretsUsed.add(message.secret);
} else if (message.verb === "CLOSE") {
conn.phase = "closed";
}
} else if (kind === "attach") {
let message: Parsed;
try {
message = parseLine(event.line, "d2n");
} catch {
continue;
}
const known = streams.get(message.stream_id);
const good = known !== undefined && sameText(known.secret, message.secret);
const fresh = good && !known.consumed && now - known.t <= pol("protocol.attach_timeout_s");
if (event.result === "ok") {
if (known === undefined) return "attach_unknown_accepted";
if (!good) return "attach_wrong_secret_accepted";
if (known.consumed) return "attach_secret_reused";
if (!fresh) return "attach_after_timeout";
known.consumed = true;
} else if (fresh) {
return "attach_refused_good_secret";
}
} else if (kind === "end") {
ended = true;
for (const other of conns.values()) {
if (other.phase === "ready" && other.superseded) return "replaced_not_closed";
if (other.phase === "ready" && now > other.validUntil) return "no_close_at_valid_until";
}
}
}
return ended ? "" : "transcript_without_end";
};
return {
pol, labelOk, fqdnOk, hostPattern, validate, parseLine, formatLine, verifySession, verifyProof, verifyNodeAssertion, verifyStatement, parseJwks,
parseNodeKeys, checkFeed, surfaceMac, surfaceProof, verifySurfaceChallenge, verifySurfaceProof, identityRequest, checkTranscript, canonicalJson,
labelRegexSource: labelRe.source,
};
}
export type Contracts = ReturnType<typeof createContracts>;
export { Reject, b64uDecode, b64uEncode };
files/contracts/vectors/bodies.json 191 lines · 68.7 KB
{
"bodies": [
{"id": "body.authorize-request.ok.baseline", "schema": "authorize-request.schema.json", "body": {"v": 1, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "session_jti": "oL0VNxP9LK9bV6LvYDEeZz", "reason": "hello", "node": "t1"}, "ok": true},
{"id": "body.authorize-request.bad.hostname_field", "schema": "authorize-request.schema.json", "body": {"v": 1, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "session_jti": "oL0VNxP9LK9bV6LvYDEeZz", "reason": "hello", "node": "t1", "hostname": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test"}, "ok": false},
{"id": "body.authorize-request.bad.bad_reason", "schema": "authorize-request.schema.json", "body": {"v": 1, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "session_jti": "oL0VNxP9LK9bV6LvYDEeZz", "reason": "reconnect", "node": "t1"}, "ok": false},
{"id": "body.authorize-request.bad.short_jti", "schema": "authorize-request.schema.json", "body": {"v": 1, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "session_jti": "short", "reason": "hello", "node": "t1"}, "ok": false},
{"id": "body.authorize-request.bad.upper_device", "schema": "authorize-request.schema.json", "body": {"v": 1, "device_id": "B290D929D06D7DD6C9A98A5B8A5276AA", "session_jti": "oL0VNxP9LK9bV6LvYDEeZz", "reason": "hello", "node": "t1"}, "ok": false},
{"id": "body.authorize-request.bad.no_node", "schema": "authorize-request.schema.json", "body": {"v": 1, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "session_jti": "oL0VNxP9LK9bV6LvYDEeZz", "reason": "hello"}, "ok": false},
{"id": "body.authorize-request.bad.proto_key", "schema": "authorize-request.schema.json", "body": {"v": 1, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "session_jti": "oL0VNxP9LK9bV6LvYDEeZz", "reason": "hello", "node": "t1", "__proto__": {}}, "ok": false},
{"id": "body.authorize-request.bad.constructor_key", "schema": "authorize-request.schema.json", "body": {"v": 1, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "session_jti": "oL0VNxP9LK9bV6LvYDEeZz", "reason": "hello", "node": "t1", "constructor": 1}, "ok": false},
{"id": "body.authorize-request.bad.tostring_key", "schema": "authorize-request.schema.json", "body": {"v": 1, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "session_jti": "oL0VNxP9LK9bV6LvYDEeZz", "reason": "hello", "node": "t1", "toString": "x"}, "ok": false},
{"id": "body.authorize-request.bad.valueof_key", "schema": "authorize-request.schema.json", "body": {"v": 1, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "session_jti": "oL0VNxP9LK9bV6LvYDEeZz", "reason": "hello", "node": "t1", "valueOf": 1}, "ok": false},
{"id": "body.authorize-response.ok.baseline", "schema": "authorize-response.schema.json", "body": {"ok": true, "v": 1, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "status": "active", "plan": "paid", "valid_until": 1798776200, "server_time": 1798761800, "policy": {"ping_interval_s": 20, "dead_peer_s": 45, "stream_idle_s": 1800, "stream_max_age_s": 14400, "renew_before_valid_until_s": 300}, "nodes": [{"id": "t1", "host": "t1.nodes.all-e2e.test", "port": 443}, {"id": "t2", "host": "t2.nodes.all-e2e.test", "port": 443, "pins": ["PWfQZKzPPgrgT_6J-s6szGhJP2_XPjZVUeaMw35EN1s"]}], "e2e_wave": 1, "e2e_kill": false, "copy_e2ee": false, "access": "eyJhbGciOiJFZERTQSIsImtpZCI6ImNwLTIwMjctMDEiLCJ0eXAiOiJkZXZpY2UtYWNjZXNzIn0.eyJ0eXAiOiJkZXZpY2UtYWNjZXNzIiwidmVyIjoxLCJpc3MiOiJodHRwczovL2FwaS5hbGwtbWNwLmZ5aSIsImF1ZCI6ImFsbC1kZXZpY2UtYWNjZXNzIiwic3ViIjoiYjI5MGQ5MjlkMDZkN2RkNmM5YTk4YTViOGE1Mjc2YWEiLCJpYXQiOjE3OTg3NjE4MDAsImV4cCI6MTc5ODc2MjQwMCwianRpIjoiYjI0b0F3SWwxOWZZN1Rac0hvMDBDYiIsIm1vZGUiOiJhY3RpdmUiLCJzZXEiOjN9.ieOnsCw_-gkrsmHnhjIA6U82ld1kiGg03DM1hjbpyvwpwobzt7Is-c-BbQsORJgiPoCPjcEYn23mc5FCXpFWBA"}, "ok": true},
{"id": "body.authorize-response.bad.bad_status", "schema": "authorize-response.schema.json", "body": {"ok": true, "v": 1, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "status": "denied", "plan": "paid", "valid_until": 1798776200, "server_time": 1798761800, "policy": {"ping_interval_s": 20, "dead_peer_s": 45, "stream_idle_s": 1800, "stream_max_age_s": 14400, "renew_before_valid_until_s": 300}, "nodes": [{"id": "t1", "host": "t1.nodes.all-e2e.test", "port": 443}, {"id": "t2", "host": "t2.nodes.all-e2e.test", "port": 443, "pins": ["PWfQZKzPPgrgT_6J-s6szGhJP2_XPjZVUeaMw35EN1s"]}], "e2e_wave": 1, "e2e_kill": false, "copy_e2ee": false, "access": "eyJhbGciOiJFZERTQSIsImtpZCI6ImNwLTIwMjctMDEiLCJ0eXAiOiJkZXZpY2UtYWNjZXNzIn0.eyJ0eXAiOiJkZXZpY2UtYWNjZXNzIiwidmVyIjoxLCJpc3MiOiJodHRwczovL2FwaS5hbGwtbWNwLmZ5aSIsImF1ZCI6ImFsbC1kZXZpY2UtYWNjZXNzIiwic3ViIjoiYjI5MGQ5MjlkMDZkN2RkNmM5YTk4YTViOGE1Mjc2YWEiLCJpYXQiOjE3OTg3NjE4MDAsImV4cCI6MTc5ODc2MjQwMCwianRpIjoiYjI0b0F3SWwxOWZZN1Rac0hvMDBDYiIsIm1vZGUiOiJhY3RpdmUiLCJzZXEiOjN9.ieOnsCw_-gkrsmHnhjIA6U82ld1kiGg03DM1hjbpyvwpwobzt7Is-c-BbQsORJgiPoCPjcEYn23mc5FCXpFWBA"}, "ok": false},
{"id": "body.authorize-response.bad.plan_gated", "schema": "authorize-response.schema.json", "body": {"ok": true, "v": 1, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "status": "active", "plan": "gated", "valid_until": 1798776200, "server_time": 1798761800, "policy": {"ping_interval_s": 20, "dead_peer_s": 45, "stream_idle_s": 1800, "stream_max_age_s": 14400, "renew_before_valid_until_s": 300}, "nodes": [{"id": "t1", "host": "t1.nodes.all-e2e.test", "port": 443}, {"id": "t2", "host": "t2.nodes.all-e2e.test", "port": 443, "pins": ["PWfQZKzPPgrgT_6J-s6szGhJP2_XPjZVUeaMw35EN1s"]}], "e2e_wave": 1, "e2e_kill": false, "copy_e2ee": false, "access": "eyJhbGciOiJFZERTQSIsImtpZCI6ImNwLTIwMjctMDEiLCJ0eXAiOiJkZXZpY2UtYWNjZXNzIn0.eyJ0eXAiOiJkZXZpY2UtYWNjZXNzIiwidmVyIjoxLCJpc3MiOiJodHRwczovL2FwaS5hbGwtbWNwLmZ5aSIsImF1ZCI6ImFsbC1kZXZpY2UtYWNjZXNzIiwic3ViIjoiYjI5MGQ5MjlkMDZkN2RkNmM5YTk4YTViOGE1Mjc2YWEiLCJpYXQiOjE3OTg3NjE4MDAsImV4cCI6MTc5ODc2MjQwMCwianRpIjoiYjI0b0F3SWwxOWZZN1Rac0hvMDBDYiIsIm1vZGUiOiJhY3RpdmUiLCJzZXEiOjN9.ieOnsCw_-gkrsmHnhjIA6U82ld1kiGg03DM1hjbpyvwpwobzt7Is-c-BbQsORJgiPoCPjcEYn23mc5FCXpFWBA"}, "ok": false},
{"id": "body.authorize-response.bad.label_underscore", "schema": "authorize-response.schema.json", "body": {"ok": true, "v": 1, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "aaaaaaaaaaaaaaaaaaa_", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "status": "active", "plan": "paid", "valid_until": 1798776200, "server_time": 1798761800, "policy": {"ping_interval_s": 20, "dead_peer_s": 45, "stream_idle_s": 1800, "stream_max_age_s": 14400, "renew_before_valid_until_s": 300}, "nodes": [{"id": "t1", "host": "t1.nodes.all-e2e.test", "port": 443}, {"id": "t2", "host": "t2.nodes.all-e2e.test", "port": 443, "pins": ["PWfQZKzPPgrgT_6J-s6szGhJP2_XPjZVUeaMw35EN1s"]}], "e2e_wave": 1, "e2e_kill": false, "copy_e2ee": false, "access": "eyJhbGciOiJFZERTQSIsImtpZCI6ImNwLTIwMjctMDEiLCJ0eXAiOiJkZXZpY2UtYWNjZXNzIn0.eyJ0eXAiOiJkZXZpY2UtYWNjZXNzIiwidmVyIjoxLCJpc3MiOiJodHRwczovL2FwaS5hbGwtbWNwLmZ5aSIsImF1ZCI6ImFsbC1kZXZpY2UtYWNjZXNzIiwic3ViIjoiYjI5MGQ5MjlkMDZkN2RkNmM5YTk4YTViOGE1Mjc2YWEiLCJpYXQiOjE3OTg3NjE4MDAsImV4cCI6MTc5ODc2MjQwMCwianRpIjoiYjI0b0F3SWwxOWZZN1Rac0hvMDBDYiIsIm1vZGUiOiJhY3RpdmUiLCJzZXEiOjN9.ieOnsCw_-gkrsmHnhjIA6U82ld1kiGg03DM1hjbpyvwpwobzt7Is-c-BbQsORJgiPoCPjcEYn23mc5FCXpFWBA"}, "ok": false},
{"id": "body.authorize-response.bad.host_uppercase", "schema": "authorize-response.schema.json", "body": {"ok": true, "v": 1, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "host": "PSRHZ6N5WPZBM6YFKNGO.DEVICES.ALL-E2E.TEST", "status": "active", "plan": "paid", "valid_until": 1798776200, "server_time": 1798761800, "policy": {"ping_interval_s": 20, "dead_peer_s": 45, "stream_idle_s": 1800, "stream_max_age_s": 14400, "renew_before_valid_until_s": 300}, "nodes": [{"id": "t1", "host": "t1.nodes.all-e2e.test", "port": 443}, {"id": "t2", "host": "t2.nodes.all-e2e.test", "port": 443, "pins": ["PWfQZKzPPgrgT_6J-s6szGhJP2_XPjZVUeaMw35EN1s"]}], "e2e_wave": 1, "e2e_kill": false, "copy_e2ee": false, "access": "eyJhbGciOiJFZERTQSIsImtpZCI6ImNwLTIwMjctMDEiLCJ0eXAiOiJkZXZpY2UtYWNjZXNzIn0.eyJ0eXAiOiJkZXZpY2UtYWNjZXNzIiwidmVyIjoxLCJpc3MiOiJodHRwczovL2FwaS5hbGwtbWNwLmZ5aSIsImF1ZCI6ImFsbC1kZXZpY2UtYWNjZXNzIiwic3ViIjoiYjI5MGQ5MjlkMDZkN2RkNmM5YTk4YTViOGE1Mjc2YWEiLCJpYXQiOjE3OTg3NjE4MDAsImV4cCI6MTc5ODc2MjQwMCwianRpIjoiYjI0b0F3SWwxOWZZN1Rac0hvMDBDYiIsIm1vZGUiOiJhY3RpdmUiLCJzZXEiOjN9.ieOnsCw_-gkrsmHnhjIA6U82ld1kiGg03DM1hjbpyvwpwobzt7Is-c-BbQsORJgiPoCPjcEYn23mc5FCXpFWBA"}, "ok": false},
{"id": "body.authorize-response.bad.zero_valid_until", "schema": "authorize-response.schema.json", "body": {"ok": true, "v": 1, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "status": "active", "plan": "paid", "valid_until": 0, "server_time": 1798761800, "policy": {"ping_interval_s": 20, "dead_peer_s": 45, "stream_idle_s": 1800, "stream_max_age_s": 14400, "renew_before_valid_until_s": 300}, "nodes": [{"id": "t1", "host": "t1.nodes.all-e2e.test", "port": 443}, {"id": "t2", "host": "t2.nodes.all-e2e.test", "port": 443, "pins": ["PWfQZKzPPgrgT_6J-s6szGhJP2_XPjZVUeaMw35EN1s"]}], "e2e_wave": 1, "e2e_kill": false, "copy_e2ee": false, "access": "eyJhbGciOiJFZERTQSIsImtpZCI6ImNwLTIwMjctMDEiLCJ0eXAiOiJkZXZpY2UtYWNjZXNzIn0.eyJ0eXAiOiJkZXZpY2UtYWNjZXNzIiwidmVyIjoxLCJpc3MiOiJodHRwczovL2FwaS5hbGwtbWNwLmZ5aSIsImF1ZCI6ImFsbC1kZXZpY2UtYWNjZXNzIiwic3ViIjoiYjI5MGQ5MjlkMDZkN2RkNmM5YTk4YTViOGE1Mjc2YWEiLCJpYXQiOjE3OTg3NjE4MDAsImV4cCI6MTc5ODc2MjQwMCwianRpIjoiYjI0b0F3SWwxOWZZN1Rac0hvMDBDYiIsIm1vZGUiOiJhY3RpdmUiLCJzZXEiOjN9.ieOnsCw_-gkrsmHnhjIA6U82ld1kiGg03DM1hjbpyvwpwobzt7Is-c-BbQsORJgiPoCPjcEYn23mc5FCXpFWBA"}, "ok": false},
{"id": "body.authorize-response.bad.no_nodes", "schema": "authorize-response.schema.json", "body": {"ok": true, "v": 1, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "status": "active", "plan": "paid", "valid_until": 1798776200, "server_time": 1798761800, "policy": {"ping_interval_s": 20, "dead_peer_s": 45, "stream_idle_s": 1800, "stream_max_age_s": 14400, "renew_before_valid_until_s": 300}, "nodes": [], "e2e_wave": 1, "e2e_kill": false, "copy_e2ee": false, "access": "eyJhbGciOiJFZERTQSIsImtpZCI6ImNwLTIwMjctMDEiLCJ0eXAiOiJkZXZpY2UtYWNjZXNzIn0.eyJ0eXAiOiJkZXZpY2UtYWNjZXNzIiwidmVyIjoxLCJpc3MiOiJodHRwczovL2FwaS5hbGwtbWNwLmZ5aSIsImF1ZCI6ImFsbC1kZXZpY2UtYWNjZXNzIiwic3ViIjoiYjI5MGQ5MjlkMDZkN2RkNmM5YTk4YTViOGE1Mjc2YWEiLCJpYXQiOjE3OTg3NjE4MDAsImV4cCI6MTc5ODc2MjQwMCwianRpIjoiYjI0b0F3SWwxOWZZN1Rac0hvMDBDYiIsIm1vZGUiOiJhY3RpdmUiLCJzZXEiOjN9.ieOnsCw_-gkrsmHnhjIA6U82ld1kiGg03DM1hjbpyvwpwobzt7Is-c-BbQsORJgiPoCPjcEYn23mc5FCXpFWBA"}, "ok": false},
{"id": "body.authorize-response.bad.wave_negative", "schema": "authorize-response.schema.json", "body": {"ok": true, "v": 1, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "status": "active", "plan": "paid", "valid_until": 1798776200, "server_time": 1798761800, "policy": {"ping_interval_s": 20, "dead_peer_s": 45, "stream_idle_s": 1800, "stream_max_age_s": 14400, "renew_before_valid_until_s": 300}, "nodes": [{"id": "t1", "host": "t1.nodes.all-e2e.test", "port": 443}, {"id": "t2", "host": "t2.nodes.all-e2e.test", "port": 443, "pins": ["PWfQZKzPPgrgT_6J-s6szGhJP2_XPjZVUeaMw35EN1s"]}], "e2e_wave": -1, "e2e_kill": false, "copy_e2ee": false, "access": "eyJhbGciOiJFZERTQSIsImtpZCI6ImNwLTIwMjctMDEiLCJ0eXAiOiJkZXZpY2UtYWNjZXNzIn0.eyJ0eXAiOiJkZXZpY2UtYWNjZXNzIiwidmVyIjoxLCJpc3MiOiJodHRwczovL2FwaS5hbGwtbWNwLmZ5aSIsImF1ZCI6ImFsbC1kZXZpY2UtYWNjZXNzIiwic3ViIjoiYjI5MGQ5MjlkMDZkN2RkNmM5YTk4YTViOGE1Mjc2YWEiLCJpYXQiOjE3OTg3NjE4MDAsImV4cCI6MTc5ODc2MjQwMCwianRpIjoiYjI0b0F3SWwxOWZZN1Rac0hvMDBDYiIsIm1vZGUiOiJhY3RpdmUiLCJzZXEiOjN9.ieOnsCw_-gkrsmHnhjIA6U82ld1kiGg03DM1hjbpyvwpwobzt7Is-c-BbQsORJgiPoCPjcEYn23mc5FCXpFWBA"}, "ok": false},
{"id": "body.authorize-response.bad.kill_text", "schema": "authorize-response.schema.json", "body": {"ok": true, "v": 1, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "status": "active", "plan": "paid", "valid_until": 1798776200, "server_time": 1798761800, "policy": {"ping_interval_s": 20, "dead_peer_s": 45, "stream_idle_s": 1800, "stream_max_age_s": 14400, "renew_before_valid_until_s": 300}, "nodes": [{"id": "t1", "host": "t1.nodes.all-e2e.test", "port": 443}, {"id": "t2", "host": "t2.nodes.all-e2e.test", "port": 443, "pins": ["PWfQZKzPPgrgT_6J-s6szGhJP2_XPjZVUeaMw35EN1s"]}], "e2e_wave": 1, "e2e_kill": "yes", "copy_e2ee": false, "access": "eyJhbGciOiJFZERTQSIsImtpZCI6ImNwLTIwMjctMDEiLCJ0eXAiOiJkZXZpY2UtYWNjZXNzIn0.eyJ0eXAiOiJkZXZpY2UtYWNjZXNzIiwidmVyIjoxLCJpc3MiOiJodHRwczovL2FwaS5hbGwtbWNwLmZ5aSIsImF1ZCI6ImFsbC1kZXZpY2UtYWNjZXNzIiwic3ViIjoiYjI5MGQ5MjlkMDZkN2RkNmM5YTk4YTViOGE1Mjc2YWEiLCJpYXQiOjE3OTg3NjE4MDAsImV4cCI6MTc5ODc2MjQwMCwianRpIjoiYjI0b0F3SWwxOWZZN1Rac0hvMDBDYiIsIm1vZGUiOiJhY3RpdmUiLCJzZXEiOjN9.ieOnsCw_-gkrsmHnhjIA6U82ld1kiGg03DM1hjbpyvwpwobzt7Is-c-BbQsORJgiPoCPjcEYn23mc5FCXpFWBA"}, "ok": false},
{"id": "body.authorize-response.bad.float_time", "schema": "authorize-response.schema.json", "body": {"ok": true, "v": 1, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "status": "active", "plan": "paid", "valid_until": 1798776200, "server_time": 1.5, "policy": {"ping_interval_s": 20, "dead_peer_s": 45, "stream_idle_s": 1800, "stream_max_age_s": 14400, "renew_before_valid_until_s": 300}, "nodes": [{"id": "t1", "host": "t1.nodes.all-e2e.test", "port": 443}, {"id": "t2", "host": "t2.nodes.all-e2e.test", "port": 443, "pins": ["PWfQZKzPPgrgT_6J-s6szGhJP2_XPjZVUeaMw35EN1s"]}], "e2e_wave": 1, "e2e_kill": false, "copy_e2ee": false, "access": "eyJhbGciOiJFZERTQSIsImtpZCI6ImNwLTIwMjctMDEiLCJ0eXAiOiJkZXZpY2UtYWNjZXNzIn0.eyJ0eXAiOiJkZXZpY2UtYWNjZXNzIiwidmVyIjoxLCJpc3MiOiJodHRwczovL2FwaS5hbGwtbWNwLmZ5aSIsImF1ZCI6ImFsbC1kZXZpY2UtYWNjZXNzIiwic3ViIjoiYjI5MGQ5MjlkMDZkN2RkNmM5YTk4YTViOGE1Mjc2YWEiLCJpYXQiOjE3OTg3NjE4MDAsImV4cCI6MTc5ODc2MjQwMCwianRpIjoiYjI0b0F3SWwxOWZZN1Rac0hvMDBDYiIsIm1vZGUiOiJhY3RpdmUiLCJzZXEiOjN9.ieOnsCw_-gkrsmHnhjIA6U82ld1kiGg03DM1hjbpyvwpwobzt7Is-c-BbQsORJgiPoCPjcEYn23mc5FCXpFWBA"}, "ok": false},
{"id": "body.authorize-response.bad.extra_key", "schema": "authorize-response.schema.json", "body": {"ok": true, "v": 1, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "status": "active", "plan": "paid", "valid_until": 1798776200, "server_time": 1798761800, "policy": {"ping_interval_s": 20, "dead_peer_s": 45, "stream_idle_s": 1800, "stream_max_age_s": 14400, "renew_before_valid_until_s": 300}, "nodes": [{"id": "t1", "host": "t1.nodes.all-e2e.test", "port": 443}, {"id": "t2", "host": "t2.nodes.all-e2e.test", "port": 443, "pins": ["PWfQZKzPPgrgT_6J-s6szGhJP2_XPjZVUeaMw35EN1s"]}], "e2e_wave": 1, "e2e_kill": false, "copy_e2ee": false, "access": "eyJhbGciOiJFZERTQSIsImtpZCI6ImNwLTIwMjctMDEiLCJ0eXAiOiJkZXZpY2UtYWNjZXNzIn0.eyJ0eXAiOiJkZXZpY2UtYWNjZXNzIiwidmVyIjoxLCJpc3MiOiJodHRwczovL2FwaS5hbGwtbWNwLmZ5aSIsImF1ZCI6ImFsbC1kZXZpY2UtYWNjZXNzIiwic3ViIjoiYjI5MGQ5MjlkMDZkN2RkNmM5YTk4YTViOGE1Mjc2YWEiLCJpYXQiOjE3OTg3NjE4MDAsImV4cCI6MTc5ODc2MjQwMCwianRpIjoiYjI0b0F3SWwxOWZZN1Rac0hvMDBDYiIsIm1vZGUiOiJhY3RpdmUiLCJzZXEiOjN9.ieOnsCw_-gkrsmHnhjIA6U82ld1kiGg03DM1hjbpyvwpwobzt7Is-c-BbQsORJgiPoCPjcEYn23mc5FCXpFWBA", "debug": true}, "ok": false},
{"id": "body.authorize-response.bad.node_port", "schema": "authorize-response.schema.json", "body": {"ok": true, "v": 1, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "status": "active", "plan": "paid", "valid_until": 1798776200, "server_time": 1798761800, "policy": {"ping_interval_s": 20, "dead_peer_s": 45, "stream_idle_s": 1800, "stream_max_age_s": 14400, "renew_before_valid_until_s": 300}, "nodes": [{"id": "t1", "host": "t1.nodes.all-e2e.test", "port": 0}], "e2e_wave": 1, "e2e_kill": false, "copy_e2ee": false, "access": "eyJhbGciOiJFZERTQSIsImtpZCI6ImNwLTIwMjctMDEiLCJ0eXAiOiJkZXZpY2UtYWNjZXNzIn0.eyJ0eXAiOiJkZXZpY2UtYWNjZXNzIiwidmVyIjoxLCJpc3MiOiJodHRwczovL2FwaS5hbGwtbWNwLmZ5aSIsImF1ZCI6ImFsbC1kZXZpY2UtYWNjZXNzIiwic3ViIjoiYjI5MGQ5MjlkMDZkN2RkNmM5YTk4YTViOGE1Mjc2YWEiLCJpYXQiOjE3OTg3NjE4MDAsImV4cCI6MTc5ODc2MjQwMCwianRpIjoiYjI0b0F3SWwxOWZZN1Rac0hvMDBDYiIsIm1vZGUiOiJhY3RpdmUiLCJzZXEiOjN9.ieOnsCw_-gkrsmHnhjIA6U82ld1kiGg03DM1hjbpyvwpwobzt7Is-c-BbQsORJgiPoCPjcEYn23mc5FCXpFWBA"}, "ok": false},
{"id": "body.authorize-response.bad.access_not_jwt", "schema": "authorize-response.schema.json", "body": {"ok": true, "v": 1, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "status": "active", "plan": "paid", "valid_until": 1798776200, "server_time": 1798761800, "policy": {"ping_interval_s": 20, "dead_peer_s": 45, "stream_idle_s": 1800, "stream_max_age_s": 14400, "renew_before_valid_until_s": 300}, "nodes": [{"id": "t1", "host": "t1.nodes.all-e2e.test", "port": 443}, {"id": "t2", "host": "t2.nodes.all-e2e.test", "port": 443, "pins": ["PWfQZKzPPgrgT_6J-s6szGhJP2_XPjZVUeaMw35EN1s"]}], "e2e_wave": 1, "e2e_kill": false, "copy_e2ee": false, "access": "plain"}, "ok": false},
{"id": "body.enroll-request.ok.baseline", "schema": "enroll-request.schema.json", "body": {"v": 1, "intent": "install"}, "ok": true},
{"id": "body.enroll-request.ok.repair", "schema": "enroll-request.schema.json", "body": {"v": 1, "intent": "repair"}, "ok": true},
{"id": "body.enroll-request.ok.reinstall", "schema": "enroll-request.schema.json", "body": {"v": 1, "intent": "reinstall"}, "ok": true},
{"id": "body.enroll-request.ok.migrate", "schema": "enroll-request.schema.json", "body": {"v": 1, "intent": "migrate"}, "ok": true},
{"id": "body.enroll-request.bad.bad_intent", "schema": "enroll-request.schema.json", "body": {"v": 1, "intent": "upgrade"}, "ok": false},
{"id": "body.enroll-request.bad.version_two", "schema": "enroll-request.schema.json", "body": {"v": 2, "intent": "install"}, "ok": false},
{"id": "body.enroll-request.bad.extra", "schema": "enroll-request.schema.json", "body": {"v": 1, "intent": "install", "label": "psrhz6n5wpzbm6yfkngo"}, "ok": false},
{"id": "body.enroll-request.bad.no_intent", "schema": "enroll-request.schema.json", "body": {"v": 1}, "ok": false},
{"id": "body.enroll-request.bad.hasownproperty_key", "schema": "enroll-request.schema.json", "body": {"v": 1, "intent": "install", "hasOwnProperty": 1}, "ok": false},
{"id": "body.enroll-request.bad.isprototypeof_key", "schema": "enroll-request.schema.json", "body": {"v": 1, "intent": "install", "isPrototypeOf": 1}, "ok": false},
{"id": "body.enroll-response.ok.baseline", "schema": "enroll-response.schema.json", "body": {"ok": true, "v": 1, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "zone": "devices.all-e2e.test", "nodes": [{"id": "t1", "host": "t1.nodes.all-e2e.test", "port": 443}, {"id": "t2", "host": "t2.nodes.all-e2e.test", "port": 443, "pins": ["PWfQZKzPPgrgT_6J-s6szGhJP2_XPjZVUeaMw35EN1s"]}], "enroll_after": null, "server_time": 1798761800, "e2e_wave": 1, "e2e_kill": false, "copy_e2ee": false}, "ok": true},
{"id": "body.enroll-response.ok.enroll_after_set", "schema": "enroll-response.schema.json", "body": {"ok": true, "v": 1, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "zone": "devices.all-e2e.test", "nodes": [{"id": "t1", "host": "t1.nodes.all-e2e.test", "port": 443}, {"id": "t2", "host": "t2.nodes.all-e2e.test", "port": 443, "pins": ["PWfQZKzPPgrgT_6J-s6szGhJP2_XPjZVUeaMw35EN1s"]}], "enroll_after": 1798761890, "server_time": 1798761800, "e2e_wave": 1, "e2e_kill": false, "copy_e2ee": false}, "ok": true},
{"id": "body.enroll-response.ok.cohort_paid", "schema": "enroll-response.schema.json", "body": {"ok": true, "v": 1, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "zone": "devices.all-e2e.test", "nodes": [{"id": "t1", "host": "t1.nodes.all-e2e.test", "port": 443}, {"id": "t2", "host": "t2.nodes.all-e2e.test", "port": 443, "pins": ["PWfQZKzPPgrgT_6J-s6szGhJP2_XPjZVUeaMw35EN1s"]}], "enroll_after": null, "server_time": 1798761800, "e2e_wave": 1, "e2e_kill": false, "copy_e2ee": false, "cohort": "paid", "eligible": true}, "ok": true},
{"id": "body.enroll-response.ok.ineligible_lifetime", "schema": "enroll-response.schema.json", "body": {"ok": true, "v": 1, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "zone": "devices.all-e2e.test", "nodes": [{"id": "t1", "host": "t1.nodes.all-e2e.test", "port": 443}, {"id": "t2", "host": "t2.nodes.all-e2e.test", "port": 443, "pins": ["PWfQZKzPPgrgT_6J-s6szGhJP2_XPjZVUeaMw35EN1s"]}], "enroll_after": 1799366600, "server_time": 1798761800, "e2e_wave": 1, "e2e_kill": false, "copy_e2ee": false, "cohort": "flex", "eligible": false, "reason": "update_not_in_plan"}, "ok": true},
{"id": "body.enroll-response.ok.legacy_credential", "schema": "enroll-response.schema.json", "body": {"ok": true, "v": 1, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "zone": "devices.all-e2e.test", "nodes": [{"id": "t1", "host": "t1.nodes.all-e2e.test", "port": 443}, {"id": "t2", "host": "t2.nodes.all-e2e.test", "port": 443, "pins": ["PWfQZKzPPgrgT_6J-s6szGhJP2_XPjZVUeaMw35EN1s"]}], "enroll_after": 1799366600, "server_time": 1798761800, "e2e_wave": 1, "e2e_kill": false, "copy_e2ee": false, "cohort": "legacy", "eligible": false, "reason": "legacy_credential"}, "ok": true},
{"id": "body.enroll-response.bad.label_short", "schema": "enroll-response.schema.json", "body": {"ok": true, "v": 1, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "abc", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "zone": "devices.all-e2e.test", "nodes": [{"id": "t1", "host": "t1.nodes.all-e2e.test", "port": 443}, {"id": "t2", "host": "t2.nodes.all-e2e.test", "port": 443, "pins": ["PWfQZKzPPgrgT_6J-s6szGhJP2_XPjZVUeaMw35EN1s"]}], "enroll_after": null, "server_time": 1798761800, "e2e_wave": 1, "e2e_kill": false, "copy_e2ee": false}, "ok": false},
{"id": "body.enroll-response.bad.host_two_labels", "schema": "enroll-response.schema.json", "body": {"ok": true, "v": 1, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "host": "x.psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "zone": "devices.all-e2e.test", "nodes": [{"id": "t1", "host": "t1.nodes.all-e2e.test", "port": 443}, {"id": "t2", "host": "t2.nodes.all-e2e.test", "port": 443, "pins": ["PWfQZKzPPgrgT_6J-s6szGhJP2_XPjZVUeaMw35EN1s"]}], "enroll_after": null, "server_time": 1798761800, "e2e_wave": 1, "e2e_kill": false, "copy_e2ee": false}, "ok": false},
{"id": "body.enroll-response.bad.zone_uppercase", "schema": "enroll-response.schema.json", "body": {"ok": true, "v": 1, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "zone": "DEVICES.ALL-E2E.TEST", "nodes": [{"id": "t1", "host": "t1.nodes.all-e2e.test", "port": 443}, {"id": "t2", "host": "t2.nodes.all-e2e.test", "port": 443, "pins": ["PWfQZKzPPgrgT_6J-s6szGhJP2_XPjZVUeaMw35EN1s"]}], "enroll_after": null, "server_time": 1798761800, "e2e_wave": 1, "e2e_kill": false, "copy_e2ee": false}, "ok": false},
{"id": "body.enroll-response.bad.enroll_after_text", "schema": "enroll-response.schema.json", "body": {"ok": true, "v": 1, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "zone": "devices.all-e2e.test", "nodes": [{"id": "t1", "host": "t1.nodes.all-e2e.test", "port": 443}, {"id": "t2", "host": "t2.nodes.all-e2e.test", "port": 443, "pins": ["PWfQZKzPPgrgT_6J-s6szGhJP2_XPjZVUeaMw35EN1s"]}], "enroll_after": "later", "server_time": 1798761800, "e2e_wave": 1, "e2e_kill": false, "copy_e2ee": false}, "ok": false},
{"id": "body.enroll-response.bad.no_wave", "schema": "enroll-response.schema.json", "body": {"ok": true, "v": 1, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "zone": "devices.all-e2e.test", "nodes": [{"id": "t1", "host": "t1.nodes.all-e2e.test", "port": 443}, {"id": "t2", "host": "t2.nodes.all-e2e.test", "port": 443, "pins": ["PWfQZKzPPgrgT_6J-s6szGhJP2_XPjZVUeaMw35EN1s"]}], "enroll_after": null, "server_time": 1798761800, "e2e_kill": false, "copy_e2ee": false}, "ok": false},
{"id": "body.enroll-response.bad.wave_big", "schema": "enroll-response.schema.json", "body": {"ok": true, "v": 1, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "zone": "devices.all-e2e.test", "nodes": [{"id": "t1", "host": "t1.nodes.all-e2e.test", "port": 443}, {"id": "t2", "host": "t2.nodes.all-e2e.test", "port": 443, "pins": ["PWfQZKzPPgrgT_6J-s6szGhJP2_XPjZVUeaMw35EN1s"]}], "enroll_after": null, "server_time": 1798761800, "e2e_wave": 101, "e2e_kill": false, "copy_e2ee": false}, "ok": false},
{"id": "body.enroll-response.bad.cohort_unknown", "schema": "enroll-response.schema.json", "body": {"ok": true, "v": 1, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "zone": "devices.all-e2e.test", "nodes": [{"id": "t1", "host": "t1.nodes.all-e2e.test", "port": 443}, {"id": "t2", "host": "t2.nodes.all-e2e.test", "port": 443, "pins": ["PWfQZKzPPgrgT_6J-s6szGhJP2_XPjZVUeaMw35EN1s"]}], "enroll_after": null, "server_time": 1798761800, "e2e_wave": 1, "e2e_kill": false, "copy_e2ee": false, "cohort": "vip"}, "ok": false},
{"id": "body.enroll-response.bad.eligible_text", "schema": "enroll-response.schema.json", "body": {"ok": true, "v": 1, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "zone": "devices.all-e2e.test", "nodes": [{"id": "t1", "host": "t1.nodes.all-e2e.test", "port": 443}, {"id": "t2", "host": "t2.nodes.all-e2e.test", "port": 443, "pins": ["PWfQZKzPPgrgT_6J-s6szGhJP2_XPjZVUeaMw35EN1s"]}], "enroll_after": null, "server_time": 1798761800, "e2e_wave": 1, "e2e_kill": false, "copy_e2ee": false, "eligible": "yes"}, "ok": false},
{"id": "body.enroll-response.bad.reason_free_text", "schema": "enroll-response.schema.json", "body": {"ok": true, "v": 1, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "zone": "devices.all-e2e.test", "nodes": [{"id": "t1", "host": "t1.nodes.all-e2e.test", "port": 443}, {"id": "t2", "host": "t2.nodes.all-e2e.test", "port": 443, "pins": ["PWfQZKzPPgrgT_6J-s6szGhJP2_XPjZVUeaMw35EN1s"]}], "enroll_after": null, "server_time": 1798761800, "e2e_wave": 1, "e2e_kill": false, "copy_e2ee": false, "reason": "Lifetime license has no updates"}, "ok": false},
{"id": "body.enroll-response.bad.reason_unknown", "schema": "enroll-response.schema.json", "body": {"ok": true, "v": 1, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "zone": "devices.all-e2e.test", "nodes": [{"id": "t1", "host": "t1.nodes.all-e2e.test", "port": 443}, {"id": "t2", "host": "t2.nodes.all-e2e.test", "port": 443, "pins": ["PWfQZKzPPgrgT_6J-s6szGhJP2_XPjZVUeaMw35EN1s"]}], "enroll_after": null, "server_time": 1798761800, "e2e_wave": 1, "e2e_kill": false, "copy_e2ee": false, "reason": "because"}, "ok": false},
{"id": "body.reconnect-state.ok.baseline", "schema": "reconnect-state.schema.json", "body": {"state": "eligible"}, "ok": true},
{"id": "body.reconnect-state.ok.migrated_since", "schema": "reconnect-state.schema.json", "body": {"state": "migrated", "since": 1798070600000}, "ok": true},
{"id": "body.reconnect-state.ok.locale_es", "schema": "reconnect-state.schema.json", "body": {"state": "eligible", "locale": "es"}, "ok": true},
{"id": "body.reconnect-state.ok.locale_en", "schema": "reconnect-state.schema.json", "body": {"state": "migrated", "since": 1797984200000, "locale": "en"}, "ok": true},
{"id": "body.reconnect-state.bad.state_other", "schema": "reconnect-state.schema.json", "body": {"state": "closed"}, "ok": false},
{"id": "body.reconnect-state.bad.since_text", "schema": "reconnect-state.schema.json", "body": {"state": "eligible", "since": "yesterday"}, "ok": false},
{"id": "body.reconnect-state.bad.since_zero", "schema": "reconnect-state.schema.json", "body": {"state": "eligible", "since": 0}, "ok": false},
{"id": "body.reconnect-state.bad.since_float", "schema": "reconnect-state.schema.json", "body": {"state": "eligible", "since": 1.5}, "ok": false},
{"id": "body.reconnect-state.bad.locale_pt", "schema": "reconnect-state.schema.json", "body": {"state": "eligible", "locale": "pt"}, "ok": false},
{"id": "body.reconnect-state.bad.extra_key", "schema": "reconnect-state.schema.json", "body": {"state": "eligible", "label": "psrhz6n5wpzbm6yfkngo"}, "ok": false},
{"id": "body.reconnect-state.bad.no_state", "schema": "reconnect-state.schema.json", "body": {}, "ok": false},
{"id": "body.selfcheck-result.ok.baseline", "schema": "selfcheck-result.schema.json", "body": {"v": 1, "ok": true, "reason": null, "tls_version": "1.3", "spki_sha256": "-WFPE9hJPPBRas8CkIZE9h5b35K9yO1t0BIHnUPrgWo", "not_after": 1802649800, "healthz": true}, "ok": true},
{"id": "body.selfcheck-result.ok.chain_failed", "schema": "selfcheck-result.schema.json", "body": {"v": 1, "ok": false, "reason": "chain", "tls_version": "1.3", "spki_sha256": null, "not_after": null, "healthz": false}, "ok": true},
{"id": "body.selfcheck-result.ok.expired", "schema": "selfcheck-result.schema.json", "body": {"v": 1, "ok": false, "reason": "expired", "tls_version": "1.3", "spki_sha256": "-WFPE9hJPPBRas8CkIZE9h5b35K9yO1t0BIHnUPrgWo", "not_after": 1798758200, "healthz": false}, "ok": true},
{"id": "body.selfcheck-result.ok.spki_mismatch", "schema": "selfcheck-result.schema.json", "body": {"v": 1, "ok": false, "reason": "spki", "tls_version": "1.3", "spki_sha256": "-WFPE9hJPPBRas8CkIZE9h5b35K9yO1t0BIHnUPrgWo", "not_after": 1802649800, "healthz": false}, "ok": true},
{"id": "body.selfcheck-result.ok.unreachable", "schema": "selfcheck-result.schema.json", "body": {"v": 1, "ok": false, "reason": "unreachable", "tls_version": null, "spki_sha256": null, "not_after": null, "healthz": false}, "ok": true},
{"id": "body.selfcheck-result.bad.reason_free_text", "schema": "selfcheck-result.schema.json", "body": {"v": 1, "ok": true, "reason": "certificate has expired for 10.0.0.1", "tls_version": "1.3", "spki_sha256": "-WFPE9hJPPBRas8CkIZE9h5b35K9yO1t0BIHnUPrgWo", "not_after": 1802649800, "healthz": true}, "ok": false},
{"id": "body.selfcheck-result.bad.reason_unknown", "schema": "selfcheck-result.schema.json", "body": {"v": 1, "ok": true, "reason": "sad", "tls_version": "1.3", "spki_sha256": "-WFPE9hJPPBRas8CkIZE9h5b35K9yO1t0BIHnUPrgWo", "not_after": 1802649800, "healthz": true}, "ok": false},
{"id": "body.selfcheck-result.bad.tls_text", "schema": "selfcheck-result.schema.json", "body": {"v": 1, "ok": true, "reason": null, "tls_version": "TLS1.3", "spki_sha256": "-WFPE9hJPPBRas8CkIZE9h5b35K9yO1t0BIHnUPrgWo", "not_after": 1802649800, "healthz": true}, "ok": false},
{"id": "body.selfcheck-result.bad.tls_one_one", "schema": "selfcheck-result.schema.json", "body": {"v": 1, "ok": true, "reason": null, "tls_version": "1.1", "spki_sha256": "-WFPE9hJPPBRas8CkIZE9h5b35K9yO1t0BIHnUPrgWo", "not_after": 1802649800, "healthz": true}, "ok": false},
{"id": "body.selfcheck-result.bad.spki_short", "schema": "selfcheck-result.schema.json", "body": {"v": 1, "ok": true, "reason": null, "tls_version": "1.3", "spki_sha256": "-WFPE9hJPPBRas8CkIZE9h5b35K9yO1t0BIHnUPrgW", "not_after": 1802649800, "healthz": true}, "ok": false},
{"id": "body.selfcheck-result.bad.healthz_text", "schema": "selfcheck-result.schema.json", "body": {"v": 1, "ok": true, "reason": null, "tls_version": "1.3", "spki_sha256": "-WFPE9hJPPBRas8CkIZE9h5b35K9yO1t0BIHnUPrgWo", "not_after": 1802649800, "healthz": "yes"}, "ok": false},
{"id": "body.selfcheck-result.bad.host_field", "schema": "selfcheck-result.schema.json", "body": {"v": 1, "ok": true, "reason": null, "tls_version": "1.3", "spki_sha256": "-WFPE9hJPPBRas8CkIZE9h5b35K9yO1t0BIHnUPrgWo", "not_after": 1802649800, "healthz": true, "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test"}, "ok": false},
{"id": "body.selfcheck-result.bad.no_ok", "schema": "selfcheck-result.schema.json", "body": {"v": 1, "reason": null, "tls_version": "1.3", "spki_sha256": "-WFPE9hJPPBRas8CkIZE9h5b35K9yO1t0BIHnUPrgWo", "not_after": 1802649800, "healthz": true}, "ok": false},
{"id": "body.selfcheck-result.bad.not_after_text", "schema": "selfcheck-result.schema.json", "body": {"v": 1, "ok": true, "reason": null, "tls_version": "1.3", "spki_sha256": "-WFPE9hJPPBRas8CkIZE9h5b35K9yO1t0BIHnUPrgWo", "not_after": "2027", "healthz": true}, "ok": false},
{"id": "body.acme-reserve-request.ok.baseline", "schema": "acme-reserve-request.schema.json", "body": {"v": 1, "spki_sha256": "-WFPE9hJPPBRas8CkIZE9h5b35K9yO1t0BIHnUPrgWo", "purpose": "first"}, "ok": true},
{"id": "body.acme-reserve-request.ok.renewal", "schema": "acme-reserve-request.schema.json", "body": {"v": 1, "spki_sha256": "-WFPE9hJPPBRas8CkIZE9h5b35K9yO1t0BIHnUPrgWo", "purpose": "renewal"}, "ok": true},
{"id": "body.acme-reserve-request.bad.spki_short", "schema": "acme-reserve-request.schema.json", "body": {"v": 1, "spki_sha256": "-WFPE9hJPPBRas8CkIZE9h5b35K9yO1t0BIHnUPrgW", "purpose": "first"}, "ok": false},
{"id": "body.acme-reserve-request.bad.spki_padded", "schema": "acme-reserve-request.schema.json", "body": {"v": 1, "spki_sha256": "-WFPE9hJPPBRas8CkIZE9h5b35K9yO1t0BIHnUPrgW=", "purpose": "first"}, "ok": false},
{"id": "body.acme-reserve-request.bad.purpose_other", "schema": "acme-reserve-request.schema.json", "body": {"v": 1, "spki_sha256": "-WFPE9hJPPBRas8CkIZE9h5b35K9yO1t0BIHnUPrgWo", "purpose": "revoke"}, "ok": false},
{"id": "body.acme-reserve-response.ok.baseline", "schema": "acme-reserve-response.schema.json", "body": {"ok": true, "v": 1, "status": "granted", "reservation_id": "c7bb8a411e3ce70146654f37484f72f1", "class": "paid", "expires_at": 1798762700, "server_time": 1798761800}, "ok": true},
{"id": "body.acme-reserve-response.bad.class_unknown", "schema": "acme-reserve-response.schema.json", "body": {"ok": true, "v": 1, "status": "granted", "reservation_id": "c7bb8a411e3ce70146654f37484f72f1", "class": "vip", "expires_at": 1798762700, "server_time": 1798761800}, "ok": false},
{"id": "body.acme-reserve-response.bad.deferred_with_id", "schema": "acme-reserve-response.schema.json", "body": {"ok": true, "v": 1, "status": "deferred", "reservation_id": "c7bb8a411e3ce70146654f37484f72f1", "class": "paid", "expires_at": 1798762700, "server_time": 1798761800}, "ok": false},
{"id": "body.acme-reserve-response.bad.reservation_upper", "schema": "acme-reserve-response.schema.json", "body": {"ok": true, "v": 1, "status": "granted", "reservation_id": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "class": "paid", "expires_at": 1798762700, "server_time": 1798761800}, "ok": false},
{"id": "body.acme-reserve-response.bad.granted_no_expiry", "schema": "acme-reserve-response.schema.json", "body": {"ok": true, "v": 1, "status": "granted", "reservation_id": "c7bb8a411e3ce70146654f37484f72f1", "class": "paid", "expires_at": null, "server_time": 1798761800}, "ok": false},
{"id": "body.acme-dns-request.ok.baseline", "schema": "acme-dns-request.schema.json", "body": {"v": 1, "reservation_id": "c7bb8a411e3ce70146654f37484f72f1", "action": "set", "value": "JFPMagGqTw_KDV9VhCKpDw-5o6HQQUEGMDScuJMERKM"}, "ok": true},
{"id": "body.acme-dns-request.bad.value_short", "schema": "acme-dns-request.schema.json", "body": {"v": 1, "reservation_id": "c7bb8a411e3ce70146654f37484f72f1", "action": "set", "value": "JFPMagGqTw_KDV9VhCKpDw-5o6HQQUEGMDScuJMERK"}, "ok": false},
{"id": "body.acme-dns-request.bad.action_other", "schema": "acme-dns-request.schema.json", "body": {"v": 1, "reservation_id": "c7bb8a411e3ce70146654f37484f72f1", "action": "delete", "value": "JFPMagGqTw_KDV9VhCKpDw-5o6HQQUEGMDScuJMERKM"}, "ok": false},
{"id": "body.acme-dns-request.bad.set_without_value", "schema": "acme-dns-request.schema.json", "body": {"v": 1, "reservation_id": "c7bb8a411e3ce70146654f37484f72f1", "action": "set"}, "ok": false},
{"id": "body.acme-dns-request.bad.record_name", "schema": "acme-dns-request.schema.json", "body": {"v": 1, "reservation_id": "c7bb8a411e3ce70146654f37484f72f1", "action": "set", "value": "JFPMagGqTw_KDV9VhCKpDw-5o6HQQUEGMDScuJMERKM", "name": "_acme-challenge.psrhz6n5wpzbm6yfkngo.devices.all-e2e.test"}, "ok": false},
{"id": "body.acme-dns-request.bad.type_field", "schema": "acme-dns-request.schema.json", "body": {"v": 1, "reservation_id": "c7bb8a411e3ce70146654f37484f72f1", "action": "set", "value": "JFPMagGqTw_KDV9VhCKpDw-5o6HQQUEGMDScuJMERKM", "type": "CNAME"}, "ok": false},
{"id": "body.acme-dns-response.ok.baseline", "schema": "acme-dns-response.schema.json", "body": {"ok": true, "v": 1, "name": "_acme-challenge.psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "type": "TXT", "ttl": 60}, "ok": true},
{"id": "body.acme-dns-response.bad.other_name", "schema": "acme-dns-response.schema.json", "body": {"ok": true, "v": 1, "name": "www.psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "type": "TXT", "ttl": 60}, "ok": false},
{"id": "body.acme-dns-response.bad.type_cname", "schema": "acme-dns-response.schema.json", "body": {"ok": true, "v": 1, "name": "_acme-challenge.psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "type": "CNAME", "ttl": 60}, "ok": false},
{"id": "body.acme-dns-response.bad.ttl_zero", "schema": "acme-dns-response.schema.json", "body": {"ok": true, "v": 1, "name": "_acme-challenge.psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "type": "TXT", "ttl": 0}, "ok": false},
{"id": "body.acme-dns-response.bad.name_other_label", "schema": "acme-dns-response.schema.json", "body": {"ok": true, "v": 1, "name": "_acme-challenge.abc.devices.all-e2e.test", "type": "TXT", "ttl": 60}, "ok": false},
{"id": "body.acme-confirm-request.ok.baseline", "schema": "acme-confirm-request.schema.json", "body": {"v": 1, "reservation_id": "c7bb8a411e3ce70146654f37484f72f1", "spki_sha256": "-WFPE9hJPPBRas8CkIZE9h5b35K9yO1t0BIHnUPrgWo", "not_before": 1798761800, "not_after": 1802649800}, "ok": true},
{"id": "body.acme-confirm-request.bad.no_after", "schema": "acme-confirm-request.schema.json", "body": {"v": 1, "reservation_id": "c7bb8a411e3ce70146654f37484f72f1", "spki_sha256": "-WFPE9hJPPBRas8CkIZE9h5b35K9yO1t0BIHnUPrgWo", "not_before": 1798761800}, "ok": false},
{"id": "body.acme-confirm-request.bad.spki_bad", "schema": "acme-confirm-request.schema.json", "body": {"v": 1, "reservation_id": "c7bb8a411e3ce70146654f37484f72f1", "spki_sha256": "short", "not_before": 1798761800, "not_after": 1802649800}, "ok": false},
{"id": "body.acme-confirm-response.ok.baseline", "schema": "acme-confirm-response.schema.json", "body": {"ok": true, "v": 1, "server_time": 1798761800}, "ok": true},
{"id": "body.acme-confirm-response.bad.not_ok", "schema": "acme-confirm-response.schema.json", "body": {"ok": false, "v": 1, "server_time": 1798761800}, "ok": false},
{"id": "body.acme-confirm-response.bad.extra", "schema": "acme-confirm-response.schema.json", "body": {"ok": true, "v": 1, "server_time": 1798761800, "recorded": true}, "ok": false},
{"id": "body.acme-release-request.ok.baseline", "schema": "acme-release-request.schema.json", "body": {"v": 1, "reason": "uninstall"}, "ok": true},
{"id": "body.acme-release-request.bad.reason_other", "schema": "acme-release-request.schema.json", "body": {"v": 1, "reason": "bored"}, "ok": false},
{"id": "body.acme-abandon-request.ok.baseline", "schema": "acme-abandon-request.schema.json", "body": {"v": 1, "reservation_id": "c7bb8a411e3ce70146654f37484f72f1"}, "ok": true},
{"id": "body.acme-abandon-request.bad.no_reservation", "schema": "acme-abandon-request.schema.json", "body": {"v": 1}, "ok": false},
{"id": "body.acme-abandon-request.bad.reservation_upper", "schema": "acme-abandon-request.schema.json", "body": {"v": 1, "reservation_id": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"}, "ok": false},
{"id": "body.acme-abandon-request.bad.reason_field", "schema": "acme-abandon-request.schema.json", "body": {"v": 1, "reservation_id": "c7bb8a411e3ce70146654f37484f72f1", "reason": "failed"}, "ok": false},
{"id": "body.acme-lapsed-response.ok.baseline", "schema": "acme-lapsed-response.schema.json", "body": {"ok": true, "v": 1, "server_time": 1798761800, "total": 2, "entries": [{"label": "psrhz6n5wpzbm6yfkngo", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "spki_sha256": "-WFPE9hJPPBRas8CkIZE9h5b35K9yO1t0BIHnUPrgWo", "ended_at": 1798589000}]}, "ok": true},
{"id": "body.acme-lapsed-response.bad.no_total", "schema": "acme-lapsed-response.schema.json", "body": {"ok": true, "v": 1, "server_time": 1798761800, "entries": [{"label": "psrhz6n5wpzbm6yfkngo", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "spki_sha256": "-WFPE9hJPPBRas8CkIZE9h5b35K9yO1t0BIHnUPrgWo", "ended_at": 1798589000}]}, "ok": false},
{"id": "body.acme-lapsed-response.bad.entry_without_key", "schema": "acme-lapsed-response.schema.json", "body": {"ok": true, "v": 1, "server_time": 1798761800, "total": 2, "entries": [{"label": "psrhz6n5wpzbm6yfkngo", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "ended_at": 1798761800}]}, "ok": false},
{"id": "body.acme-lapsed-response.bad.txt_leak", "schema": "acme-lapsed-response.schema.json", "body": {"ok": true, "v": 1, "server_time": 1798761800, "total": 2, "entries": [{"label": "psrhz6n5wpzbm6yfkngo", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "spki_sha256": "-WFPE9hJPPBRas8CkIZE9h5b35K9yO1t0BIHnUPrgWo", "ended_at": 1798761800, "txt": "JFPMagGqTw_KDV9VhCKpDw-5o6HQQUEGMDScuJMERKM"}]}, "ok": false},
{"id": "body.acme-lift-request.ok.baseline", "schema": "acme-lift-request.schema.json", "body": {"v": 1, "spki_sha256": ["-WFPE9hJPPBRas8CkIZE9h5b35K9yO1t0BIHnUPrgWo", "4VRE51m6mHb-cVSbUGIvvs1PKvPtqrZmyJHjvPnB8hs"]}, "ok": true},
{"id": "body.acme-lift-request.bad.empty", "schema": "acme-lift-request.schema.json", "body": {"v": 1, "spki_sha256": []}, "ok": false},
{"id": "body.acme-lift-request.bad.short_key", "schema": "acme-lift-request.schema.json", "body": {"v": 1, "spki_sha256": ["short"]}, "ok": false},
{"id": "body.acme-lift-request.bad.not_a_list", "schema": "acme-lift-request.schema.json", "body": {"v": 1, "spki_sha256": "-WFPE9hJPPBRas8CkIZE9h5b35K9yO1t0BIHnUPrgWo"}, "ok": false},
{"id": "body.acme-lift-request.bad.label_field", "schema": "acme-lift-request.schema.json", "body": {"v": 1, "spki_sha256": ["-WFPE9hJPPBRas8CkIZE9h5b35K9yO1t0BIHnUPrgWo", "4VRE51m6mHb-cVSbUGIvvs1PKvPtqrZmyJHjvPnB8hs"], "label": "psrhz6n5wpzbm6yfkngo"}, "ok": false},
{"id": "body.acme-lift-response.ok.baseline", "schema": "acme-lift-response.schema.json", "body": {"ok": true, "v": 1, "server_time": 1798761800, "lifted": 1}, "ok": true},
{"id": "body.acme-lift-response.bad.negative", "schema": "acme-lift-response.schema.json", "body": {"ok": true, "v": 1, "server_time": 1798761800, "lifted": -1}, "ok": false},
{"id": "body.acme-lift-response.bad.no_count", "schema": "acme-lift-response.schema.json", "body": {"ok": true, "v": 1, "server_time": 1798761800}, "ok": false},
{"id": "body.cert-status-response.ok.baseline", "schema": "cert-status-response.schema.json", "body": {"ok": true, "v": 1, "label": "psrhz6n5wpzbm6yfkngo", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "class": "trial", "reservation": null, "issuances": [{"spki_sha256": "-WFPE9hJPPBRas8CkIZE9h5b35K9yO1t0BIHnUPrgWo", "not_before": 1798761800, "not_after": 1802649800}], "enroll_after": null, "server_time": 1798761800}, "ok": true},
{"id": "body.cert-status-response.ok.reservation_open", "schema": "cert-status-response.schema.json", "body": {"ok": true, "v": 1, "label": "psrhz6n5wpzbm6yfkngo", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "class": "trial", "reservation": {"reservation_id": "c7bb8a411e3ce70146654f37484f72f1", "spki_sha256": "-WFPE9hJPPBRas8CkIZE9h5b35K9yO1t0BIHnUPrgWo", "expires_at": 1798762700}, "issuances": [{"spki_sha256": "-WFPE9hJPPBRas8CkIZE9h5b35K9yO1t0BIHnUPrgWo", "not_before": 1798761800, "not_after": 1802649800}], "enroll_after": null, "server_time": 1798761800}, "ok": true},
{"id": "body.cert-status-response.bad.reservation_partial", "schema": "cert-status-response.schema.json", "body": {"ok": true, "v": 1, "label": "psrhz6n5wpzbm6yfkngo", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "class": "trial", "reservation": {"reservation_id": "x"}, "issuances": [{"spki_sha256": "-WFPE9hJPPBRas8CkIZE9h5b35K9yO1t0BIHnUPrgWo", "not_before": 1798761800, "not_after": 1802649800}], "enroll_after": null, "server_time": 1798761800}, "ok": false},
{"id": "body.cert-status-response.bad.txt_value_leak", "schema": "cert-status-response.schema.json", "body": {"ok": true, "v": 1, "label": "psrhz6n5wpzbm6yfkngo", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "class": "trial", "reservation": null, "issuances": [{"spki_sha256": "-WFPE9hJPPBRas8CkIZE9h5b35K9yO1t0BIHnUPrgWo", "not_before": 1798761800, "not_after": 1802649800}], "enroll_after": null, "server_time": 1798761800, "txt": "JFPMagGqTw_KDV9VhCKpDw-5o6HQQUEGMDScuJMERKM"}, "ok": false},
{"id": "body.fleet-e2e-block.ok.baseline", "schema": "fleet-e2e-block.schema.json", "body": {"v": 1, "state": "ready", "cert_not_after": 1802217800, "nodes_connected": 2, "nodes_configured": 2, "last_error": null, "clock_skew": false, "ct_alert": false, "migrated": true, "cohort": "paid", "eligible": true, "window": "5c1f0a9e3b7d2468", "bytes_in": 1234, "bytes_out": 5678, "period_s": 86400}, "ok": true},
{"id": "body.fleet-e2e-block.bad.label_present", "schema": "fleet-e2e-block.schema.json", "body": {"v": 1, "state": "ready", "cert_not_after": 1802217800, "nodes_connected": 2, "nodes_configured": 2, "last_error": null, "clock_skew": false, "ct_alert": false, "migrated": true, "cohort": "paid", "eligible": true, "window": "5c1f0a9e3b7d2468", "bytes_in": 1234, "bytes_out": 5678, "period_s": 86400, "label": "psrhz6n5wpzbm6yfkngo"}, "ok": false},
{"id": "body.fleet-e2e-block.bad.device_id_present", "schema": "fleet-e2e-block.schema.json", "body": {"v": 1, "state": "ready", "cert_not_after": 1802217800, "nodes_connected": 2, "nodes_configured": 2, "last_error": null, "clock_skew": false, "ct_alert": false, "migrated": true, "cohort": "paid", "eligible": true, "window": "5c1f0a9e3b7d2468", "bytes_in": 1234, "bytes_out": 5678, "period_s": 86400, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa"}, "ok": false},
{"id": "body.fleet-e2e-block.bad.state_unknown", "schema": "fleet-e2e-block.schema.json", "body": {"v": 1, "state": "great", "cert_not_after": 1802217800, "nodes_connected": 2, "nodes_configured": 2, "last_error": null, "clock_skew": false, "ct_alert": false, "migrated": true, "cohort": "paid", "eligible": true, "window": "5c1f0a9e3b7d2468", "bytes_in": 1234, "bytes_out": 5678, "period_s": 86400}, "ok": false},
{"id": "body.fleet-e2e-block.bad.cohort_unknown", "schema": "fleet-e2e-block.schema.json", "body": {"v": 1, "state": "ready", "cert_not_after": 1802217800, "nodes_connected": 2, "nodes_configured": 2, "last_error": null, "clock_skew": false, "ct_alert": false, "migrated": true, "cohort": "vip", "eligible": true, "window": "5c1f0a9e3b7d2468", "bytes_in": 1234, "bytes_out": 5678, "period_s": 86400}, "ok": false},
{"id": "body.fleet-e2e-block.bad.bytes_negative", "schema": "fleet-e2e-block.schema.json", "body": {"v": 1, "state": "ready", "cert_not_after": 1802217800, "nodes_connected": 2, "nodes_configured": 2, "last_error": null, "clock_skew": false, "ct_alert": false, "migrated": true, "cohort": "paid", "eligible": true, "window": "5c1f0a9e3b7d2468", "bytes_in": -1, "bytes_out": 5678, "period_s": 86400}, "ok": false},
{"id": "body.fleet-e2e-block.bad.error_free_text", "schema": "fleet-e2e-block.schema.json", "body": {"v": 1, "state": "ready", "cert_not_after": 1802217800, "nodes_connected": 2, "nodes_configured": 2, "last_error": "Connection refused by 10.0.0.1", "clock_skew": false, "ct_alert": false, "migrated": true, "cohort": "paid", "eligible": true, "window": "5c1f0a9e3b7d2468", "bytes_in": 1234, "bytes_out": 5678, "period_s": 86400}, "ok": false},
{"id": "body.fleet-e2e-block.bad.nodes_many", "schema": "fleet-e2e-block.schema.json", "body": {"v": 1, "state": "ready", "cert_not_after": 1802217800, "nodes_connected": 9, "nodes_configured": 2, "last_error": null, "clock_skew": false, "ct_alert": false, "migrated": true, "cohort": "paid", "eligible": true, "window": "5c1f0a9e3b7d2468", "bytes_in": 1234, "bytes_out": 5678, "period_s": 86400}, "ok": false},
{"id": "body.fleet-e2e-block.bad.no_period", "schema": "fleet-e2e-block.schema.json", "body": {"v": 1, "state": "ready", "cert_not_after": 1802217800, "nodes_connected": 2, "nodes_configured": 2, "last_error": null, "clock_skew": false, "ct_alert": false, "migrated": true, "cohort": "paid", "eligible": true, "window": "5c1f0a9e3b7d2468", "bytes_in": 1234, "bytes_out": 5678}, "ok": false},
{"id": "body.fleet-e2e-block.bad.cert_text", "schema": "fleet-e2e-block.schema.json", "body": {"v": 1, "state": "ready", "cert_not_after": "2027", "nodes_connected": 2, "nodes_configured": 2, "last_error": null, "clock_skew": false, "ct_alert": false, "migrated": true, "cohort": "paid", "eligible": true, "window": "5c1f0a9e3b7d2468", "bytes_in": 1234, "bytes_out": 5678, "period_s": 86400}, "ok": false},
{"id": "body.fleet-e2e-block.bad.no_window", "schema": "fleet-e2e-block.schema.json", "body": {"v": 1, "state": "ready", "cert_not_after": 1802217800, "nodes_connected": 2, "nodes_configured": 2, "last_error": null, "clock_skew": false, "ct_alert": false, "migrated": true, "cohort": "paid", "eligible": true, "bytes_in": 1234, "bytes_out": 5678, "period_s": 86400}, "ok": false},
{"id": "body.fleet-e2e-block.bad.window_free_text", "schema": "fleet-e2e-block.schema.json", "body": {"v": 1, "state": "ready", "cert_not_after": 1802217800, "nodes_connected": 2, "nodes_configured": 2, "last_error": null, "clock_skew": false, "ct_alert": false, "migrated": true, "cohort": "paid", "eligible": true, "window": "laptop de Ana", "bytes_in": 1234, "bytes_out": 5678, "period_s": 86400}, "ok": false},
{"id": "body.fleet-e2e-block.bad.window_uppercase", "schema": "fleet-e2e-block.schema.json", "body": {"v": 1, "state": "ready", "cert_not_after": 1802217800, "nodes_connected": 2, "nodes_configured": 2, "last_error": null, "clock_skew": false, "ct_alert": false, "migrated": true, "cohort": "paid", "eligible": true, "window": "5C1F0A9E3B7D2468", "bytes_in": 1234, "bytes_out": 5678, "period_s": 86400}, "ok": false},
{"id": "body.surfaces-file.ok.baseline", "schema": "surfaces-file.schema.json", "body": {"v": 1, "pid": 4242, "host": "127.0.0.1", "port": 45123, "secret": "1E0RNbrxcukCk0rTUV8cQJ80BKH4PeJ228NtNdk9v7Q", "started_at": 1798761800}, "ok": true},
{"id": "body.surfaces-file.bad.host_other", "schema": "surfaces-file.schema.json", "body": {"v": 1, "pid": 4242, "host": "0.0.0.0", "port": 45123, "secret": "1E0RNbrxcukCk0rTUV8cQJ80BKH4PeJ228NtNdk9v7Q", "started_at": 1798761800}, "ok": false},
{"id": "body.surfaces-file.bad.port_low", "schema": "surfaces-file.schema.json", "body": {"v": 1, "pid": 4242, "host": "127.0.0.1", "port": 80, "secret": "1E0RNbrxcukCk0rTUV8cQJ80BKH4PeJ228NtNdk9v7Q", "started_at": 1798761800}, "ok": false},
{"id": "body.surfaces-file.bad.secret_short", "schema": "surfaces-file.schema.json", "body": {"v": 1, "pid": 4242, "host": "127.0.0.1", "port": 45123, "secret": "abc", "started_at": 1798761800}, "ok": false},
{"id": "body.surfaces-file.bad.pid_zero", "schema": "surfaces-file.schema.json", "body": {"v": 1, "pid": 0, "host": "127.0.0.1", "port": 45123, "secret": "1E0RNbrxcukCk0rTUV8cQJ80BKH4PeJ228NtNdk9v7Q", "started_at": 1798761800}, "ok": false},
{"id": "body.surfaces-file.bad.token_extra", "schema": "surfaces-file.schema.json", "body": {"v": 1, "pid": 4242, "host": "127.0.0.1", "port": 45123, "secret": "1E0RNbrxcukCk0rTUV8cQJ80BKH4PeJ228NtNdk9v7Q", "started_at": 1798761800, "token": "x"}, "ok": false},
{"id": "body.ok-body.ok.baseline", "schema": "ok-body.schema.json", "body": {"v": 2, "node": "t1", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "status": "active", "valid_until": 1798762700, "server_time": 1798761800, "policy": {"ping_interval_s": 20, "dead_peer_s": 45, "stream_idle_s": 1800, "stream_max_age_s": 14400, "renew_before_valid_until_s": 300}}, "ok": true},
{"id": "body.ok-body.bad.policy_missing_key", "schema": "ok-body.schema.json", "body": {"v": 2, "node": "t1", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "status": "active", "valid_until": 1798762700, "server_time": 1798761800, "policy": {"ping_interval_s": 20}}, "ok": false},
{"id": "body.ok-body.bad.node_upper", "schema": "ok-body.schema.json", "body": {"v": 2, "node": "T1", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "status": "active", "valid_until": 1798762700, "server_time": 1798761800, "policy": {"ping_interval_s": 20, "dead_peer_s": 45, "stream_idle_s": 1800, "stream_max_age_s": 14400, "renew_before_valid_until_s": 300}}, "ok": false},
{"id": "body.ok-body.bad.policy_window_over_a_day", "schema": "ok-body.schema.json", "body": {"v": 2, "node": "t1", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "status": "active", "valid_until": 1798762700, "server_time": 1798761800, "policy": {"ping_interval_s": 20, "dead_peer_s": 45, "stream_idle_s": 86401, "stream_max_age_s": 14400, "renew_before_valid_until_s": 300}}, "ok": false},
{"id": "body.error-response.ok.baseline", "schema": "error-response.schema.json", "body": {"ok": false, "error": "license_inactive", "status": "trial_ended"}, "ok": true},
{"id": "body.error-response.bad.ok_true", "schema": "error-response.schema.json", "body": {"ok": true, "error": "license_inactive", "status": "trial_ended"}, "ok": false},
{"id": "body.error-response.bad.text_error", "schema": "error-response.schema.json", "body": {"ok": false, "error": "License is not active", "status": "trial_ended"}, "ok": false},
{"id": "body.error-response.bad.retry_zero", "schema": "error-response.schema.json", "body": {"ok": false, "error": "license_inactive", "status": "trial_ended", "retry_after_s": 0}, "ok": false},
{"id": "body.error-response.bad.stack", "schema": "error-response.schema.json", "body": {"ok": false, "error": "license_inactive", "status": "trial_ended", "stack": "at x"}, "ok": false},
{"id": "body.jwks.ok.baseline", "schema": "jwks.schema.json", "body": {"keys": [{"kty": "OKP", "crv": "Ed25519", "x": "fX5C9uHGorCmQCuwcraTTxZPpbxUh3elPZt5uX3OvqA", "kid": "cp-2027-01", "alg": "EdDSA", "use": "sig", "key_ops": ["verify"]}, {"kty": "OKP", "crv": "Ed25519", "x": "sWrWvn1edtMmmO_FjDMCyue2t9Y_tf_7eFSYrI1vBr0", "kid": "cp-2026-10", "alg": "EdDSA", "use": "sig", "key_ops": ["verify"]}]}, "ok": true},
{"id": "body.jwks.bad.no_kid", "schema": "jwks.schema.json", "body": {"keys": [{"kty": "OKP", "crv": "Ed25519", "x": "fX5C9uHGorCmQCuwcraTTxZPpbxUh3elPZt5uX3OvqA", "alg": "EdDSA", "use": "sig", "key_ops": ["verify"]}]}, "ok": false},
{"id": "body.jwks.bad.private_part", "schema": "jwks.schema.json", "body": {"keys": [{"kty": "OKP", "crv": "Ed25519", "x": "fX5C9uHGorCmQCuwcraTTxZPpbxUh3elPZt5uX3OvqA", "kid": "cp-2027-01", "alg": "EdDSA", "use": "sig", "key_ops": ["verify"], "d": "H-5Yi1WJ5IRuX6PUB-rWFvRtcycVMNOs7Y_-J_DBCh4"}]}, "ok": false},
{"id": "body.jwks.bad.empty", "schema": "jwks.schema.json", "body": {"keys": []}, "ok": false},
{"id": "body.node-keys.ok.baseline", "schema": "node-keys.schema.json", "body": {"keys": [{"kid": "node-t1-1", "node": "t1", "kty": "OKP", "crv": "Ed25519", "x": "Pur4yosK8_cuGyraj4T-gI_Zxkb1wYrL1wNJqsK-50Q"}, {"kid": "node-t2-1", "node": "t2", "kty": "OKP", "crv": "Ed25519", "x": "l73wsYm8_aBD6pz-Aj-_tgdOcgyaZYreC9wIP-a-UcA"}]}, "ok": true},
{"id": "body.node-keys.bad.empty", "schema": "node-keys.schema.json", "body": {"keys": []}, "ok": false},
{"id": "body.node-keys.bad.no_node", "schema": "node-keys.schema.json", "body": {"keys": [{"kid": "node-t1-1", "kty": "OKP", "crv": "Ed25519", "x": "Pur4yosK8_cuGyraj4T-gI_Zxkb1wYrL1wNJqsK-50Q"}]}, "ok": false},
{"id": "body.paywall.ok.baseline", "schema": "paywall.schema.json", "body": {"version": 1, "status": "trial_ended", "plan": "trial", "trial_ended_at": 1798762100000, "locale": "es", "upgrade_url": "https://all-mcp.fyi/upgrade/?lang=es#upg1.lic_trial_0123456789abcdef01234567.1763300000.AAAAAAAAAAAAAAAAAAAAAA", "plans": [{"id": "monthly", "label": "ALL Mensual", "amount": "$3.99", "cadence": "monthly", "updates": true, "computers": 3}, {"id": "no_updates", "label": "ALL Lifetime", "amount": "$19.99", "cadence": "one_time", "updates": false, "computers": 3}, {"id": "with_updates", "label": "ALL Lifetime + Updates", "amount": "$39.99", "cadence": "one_time", "updates": true, "computers": 3}], "note_es": "Nota para el asistente.", "note_en": "Note for the assistant.", "text_es": "Tu prueba termino.", "text_en": "Your trial ended."}, "ok": true},
{"id": "body.paywall.bad.two_plans", "schema": "paywall.schema.json", "body": {"version": 1, "status": "trial_ended", "plan": "trial", "trial_ended_at": 1798762100000, "locale": "es", "upgrade_url": "https://all-mcp.fyi/upgrade/?lang=es#upg1.lic_trial_0123456789abcdef01234567.1763300000.AAAAAAAAAAAAAAAAAAAAAA", "plans": [{"id": "monthly", "label": "ALL Mensual", "amount": "$3.99", "cadence": "monthly", "updates": true, "computers": 3}, {"id": "no_updates", "label": "ALL Lifetime", "amount": "$19.99", "cadence": "one_time", "updates": false, "computers": 3}], "note_es": "Nota para el asistente.", "note_en": "Note for the assistant.", "text_es": "Tu prueba termino.", "text_en": "Your trial ended."}, "ok": false},
{"id": "body.paywall.bad.bad_url", "schema": "paywall.schema.json", "body": {"version": 1, "status": "trial_ended", "plan": "trial", "trial_ended_at": 1798762100000, "locale": "es", "upgrade_url": "https://evil.example/upgrade/#abcdefgh", "plans": [{"id": "monthly", "label": "ALL Mensual", "amount": "$3.99", "cadence": "monthly", "updates": true, "computers": 3}, {"id": "no_updates", "label": "ALL Lifetime", "amount": "$19.99", "cadence": "one_time", "updates": false, "computers": 3}, {"id": "with_updates", "label": "ALL Lifetime + Updates", "amount": "$39.99", "cadence": "one_time", "updates": true, "computers": 3}], "note_es": "Nota para el asistente.", "note_en": "Note for the assistant.", "text_es": "Tu prueba termino.", "text_en": "Your trial ended."}, "ok": false},
{"id": "body.paywall.bad.amount_text", "schema": "paywall.schema.json", "body": {"version": 1, "status": "trial_ended", "plan": "trial", "trial_ended_at": 1798762100000, "locale": "es", "upgrade_url": "https://all-mcp.fyi/upgrade/?lang=es#upg1.lic_trial_0123456789abcdef01234567.1763300000.AAAAAAAAAAAAAAAAAAAAAA", "plans": [{"id": "monthly", "label": "ALL Mensual", "amount": "free", "cadence": "monthly", "updates": true, "computers": 3}, {"id": "no_updates", "label": "ALL Lifetime", "amount": "$19.99", "cadence": "one_time", "updates": false, "computers": 3}, {"id": "with_updates", "label": "ALL Lifetime + Updates", "amount": "$39.99", "cadence": "one_time", "updates": true, "computers": 3}], "note_es": "Nota para el asistente.", "note_en": "Note for the assistant.", "text_es": "Tu prueba termino.", "text_en": "Your trial ended."}, "ok": false},
{"id": "body.paywall.bad.locale_pt", "schema": "paywall.schema.json", "body": {"version": 1, "status": "trial_ended", "plan": "trial", "trial_ended_at": 1798762100000, "locale": "pt", "upgrade_url": "https://all-mcp.fyi/upgrade/?lang=es#upg1.lic_trial_0123456789abcdef01234567.1763300000.AAAAAAAAAAAAAAAAAAAAAA", "plans": [{"id": "monthly", "label": "ALL Mensual", "amount": "$3.99", "cadence": "monthly", "updates": true, "computers": 3}, {"id": "no_updates", "label": "ALL Lifetime", "amount": "$19.99", "cadence": "one_time", "updates": false, "computers": 3}, {"id": "with_updates", "label": "ALL Lifetime + Updates", "amount": "$39.99", "cadence": "one_time", "updates": true, "computers": 3}], "note_es": "Nota para el asistente.", "note_en": "Note for the assistant.", "text_es": "Tu prueba termino.", "text_en": "Your trial ended."}, "ok": false},
{"id": "body.paywall.bad.wrong_status", "schema": "paywall.schema.json", "body": {"version": 1, "status": "active", "plan": "trial", "trial_ended_at": 1798762100000, "locale": "es", "upgrade_url": "https://all-mcp.fyi/upgrade/?lang=es#upg1.lic_trial_0123456789abcdef01234567.1763300000.AAAAAAAAAAAAAAAAAAAAAA", "plans": [{"id": "monthly", "label": "ALL Mensual", "amount": "$3.99", "cadence": "monthly", "updates": true, "computers": 3}, {"id": "no_updates", "label": "ALL Lifetime", "amount": "$19.99", "cadence": "one_time", "updates": false, "computers": 3}, {"id": "with_updates", "label": "ALL Lifetime + Updates", "amount": "$39.99", "cadence": "one_time", "updates": true, "computers": 3}], "note_es": "Nota para el asistente.", "note_en": "Note for the assistant.", "text_es": "Tu prueba termino.", "text_en": "Your trial ended."}, "ok": false},
{"id": "body.paywall.bad.plan_id", "schema": "paywall.schema.json", "body": {"version": 1, "status": "trial_ended", "plan": "trial", "trial_ended_at": 1798762100000, "locale": "es", "upgrade_url": "https://all-mcp.fyi/upgrade/?lang=es#upg1.lic_trial_0123456789abcdef01234567.1763300000.AAAAAAAAAAAAAAAAAAAAAA", "plans": [{"id": "lifetime", "label": "ALL Mensual", "amount": "$3.99", "cadence": "monthly", "updates": true, "computers": 3}, {"id": "no_updates", "label": "ALL Lifetime", "amount": "$19.99", "cadence": "one_time", "updates": false, "computers": 3}, {"id": "with_updates", "label": "ALL Lifetime + Updates", "amount": "$39.99", "cadence": "one_time", "updates": true, "computers": 3}], "note_es": "Nota para el asistente.", "note_en": "Note for the assistant.", "text_es": "Tu prueba termino.", "text_en": "Your trial ended."}, "ok": false},
{"id": "body.acme-reserve-response.ok.deferred", "schema": "acme-reserve-response.schema.json", "body": {"ok": true, "v": 1, "status": "deferred", "enroll_after": 1798765400, "server_time": 1798761800}, "ok": true},
{"id": "body.acme-dns-request.ok.clear", "schema": "acme-dns-request.schema.json", "body": {"v": 1, "reservation_id": "c7bb8a411e3ce70146654f37484f72f1", "action": "clear"}, "ok": true},
{"id": "body.acme-dns-request.bad.clear_with_value", "schema": "acme-dns-request.schema.json", "body": {"v": 1, "reservation_id": "c7bb8a411e3ce70146654f37484f72f1", "action": "clear", "value": "JFPMagGqTw_KDV9VhCKpDw-5o6HQQUEGMDScuJMERKM"}, "ok": false},
{"id": "body.authorize-response.ok.gated", "schema": "authorize-response.schema.json", "body": {"ok": true, "v": 1, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "status": "gated", "plan": "trial", "valid_until": 1798762700, "server_time": 1798761800, "policy": {"ping_interval_s": 20, "dead_peer_s": 45, "stream_idle_s": 1800, "stream_max_age_s": 14400, "renew_before_valid_until_s": 300}, "nodes": [{"id": "t1", "host": "t1.nodes.all-e2e.test", "port": 443}, {"id": "t2", "host": "t2.nodes.all-e2e.test", "port": 443, "pins": ["PWfQZKzPPgrgT_6J-s6szGhJP2_XPjZVUeaMw35EN1s"]}], "e2e_wave": 1, "e2e_kill": false, "copy_e2ee": false}, "ok": true},
{"id": "body.authorize-response.ok.paused", "schema": "authorize-response.schema.json", "body": {"ok": true, "v": 1, "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "status": "paused", "plan": "paid", "valid_until": 1798762700, "server_time": 1798761800, "policy": {"ping_interval_s": 20, "dead_peer_s": 45, "stream_idle_s": 1800, "stream_max_age_s": 14400, "renew_before_valid_until_s": 300}, "nodes": [{"id": "t1", "host": "t1.nodes.all-e2e.test", "port": 443}, {"id": "t2", "host": "t2.nodes.all-e2e.test", "port": 443, "pins": ["PWfQZKzPPgrgT_6J-s6szGhJP2_XPjZVUeaMw35EN1s"]}], "e2e_wave": 1, "e2e_kill": false, "copy_e2ee": false}, "ok": true},
{"id": "body.error-response.ok.retry", "schema": "error-response.schema.json", "body": {"ok": false, "error": "control_unavailable", "retry_after_s": 30}, "ok": true},
{"id": "body.statement-claims.ok.paid", "schema": "statement-claims.schema.json", "body": {"typ": "device-access", "ver": 1, "iss": "https://api.all-mcp.fyi", "aud": "all-device-access", "sub": "b290d929d06d7dd6c9a98a5b8a5276aa", "iat": 1798761800, "exp": 1798776200, "jti": "IJtqz0SoavLr6ck6zNPr-T", "mode": "active", "seq": 9}, "ok": true},
{"id": "body.statement-claims.bad.no_seq", "schema": "statement-claims.schema.json", "body": {"typ": "device-access", "ver": 1, "iss": "https://api.all-mcp.fyi", "aud": "all-device-access", "sub": "b290d929d06d7dd6c9a98a5b8a5276aa", "iat": 1798761800, "exp": 1798762400, "jti": "IJtqz0SoavLr6ck6zNPr-T", "mode": "active"}, "ok": false}
],
"feeds": [
{"id": "feed.ok.events", "since": 10, "ok": true, "body": {"ok": true, "v": 1, "epoch": 13, "server_time": 1798761800, "events": [{"epoch": 11, "type": "pause", "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "at": 1798761795}, {"epoch": 12, "type": "revoke", "device_id": "ac8fb81efc2ce011a6ded256c40a1d3a", "label": "zoplfhior7zh6nwqxs6q", "at": 1798761796}, {"epoch": 13, "type": "kill", "device_id": null, "label": null, "at": 1798761799}]}},
{"id": "feed.ok.empty", "since": 40, "ok": true, "body": {"ok": true, "v": 1, "epoch": 40, "server_time": 1798761800, "events": []}},
{"id": "feed.ok.reset", "since": 3, "ok": true, "body": {"ok": true, "v": 1, "epoch": 90, "server_time": 1798761800, "reset": true, "events": []}},
{"id": "feed.ok.gap_in_epochs", "since": 10, "ok": true, "body": {"ok": true, "v": 1, "epoch": 30, "server_time": 1798761800, "events": [{"epoch": 12, "type": "resume", "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "at": 1798761800}, {"epoch": 30, "type": "release", "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "at": 1798761800}]}},
{"id": "feed.bad.duplicate", "since": 10, "ok": false, "body": {"ok": true, "v": 1, "epoch": 12, "server_time": 1798761800, "events": [{"epoch": 12, "type": "pause", "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "at": 1798761800}, {"epoch": 12, "type": "pause", "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "at": 1798761800}]}},
{"id": "feed.bad.reordered", "since": 10, "ok": false, "body": {"ok": true, "v": 1, "epoch": 13, "server_time": 1798761800, "events": [{"epoch": 13, "type": "pause", "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "at": 1798761800}, {"epoch": 12, "type": "resume", "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "at": 1798761800}]}},
{"id": "feed.bad.not_after_since", "since": 12, "ok": false, "body": {"ok": true, "v": 1, "epoch": 12, "server_time": 1798761800, "events": [{"epoch": 12, "type": "pause", "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "at": 1798761800}]}},
{"id": "feed.bad.kill_with_device", "since": 1, "ok": false, "body": {"ok": true, "v": 1, "epoch": 2, "server_time": 1798761800, "events": [{"epoch": 2, "type": "kill", "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "at": 1798761800}]}},
{"id": "feed.bad.revoke_without_label", "since": 1, "ok": false, "body": {"ok": true, "v": 1, "epoch": 2, "server_time": 1798761800, "events": [{"epoch": 2, "type": "revoke", "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": null, "at": 1798761800}]}},
{"id": "feed.bad.revoke_global", "since": 1, "ok": false, "body": {"ok": true, "v": 1, "epoch": 2, "server_time": 1798761800, "events": [{"epoch": 2, "type": "revoke", "device_id": null, "label": null, "at": 1798761800}]}},
{"id": "feed.bad.epoch_behind", "since": 1, "ok": false, "body": {"ok": true, "v": 1, "epoch": 5, "server_time": 1798761800, "events": [{"epoch": 7, "type": "pause", "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "at": 1798761800}]}},
{"id": "feed.bad.unknown_type", "since": 1, "ok": false, "body": {"ok": true, "v": 1, "epoch": 2, "server_time": 1798761800, "events": [{"epoch": 2, "type": "ban", "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "at": 1798761800}]}},
{"id": "feed.bad.event_key", "since": 1, "ok": false, "body": {"ok": true, "v": 1, "epoch": 2, "server_time": 1798761800, "events": [{"epoch": 2, "type": "pause", "device_id": "b290d929d06d7dd6c9a98a5b8a5276aa", "label": "psrhz6n5wpzbm6yfkngo", "at": 1798761800, "ip": "203.0.113.5"}]}}
]
}
files/contracts/vectors/keys.json 114 lines · 3.4 KB
{
"note": "TEST KEYS ONLY. Every seed is sha256('all-contracts-v1/' + name); nothing here protects anything. Ed25519 signatures are deterministic, so tokens minted from these seeds are reproducible. cp_rogue is never published; node_rogue (node-t1-0) was revoked.",
"salt": "",
"base_time": 1798761600,
"zone": "devices.all-e2e.test",
"node_zone": "nodes.all-e2e.test",
"control_plane": {
"cp": {
"seed_hex": "38a8bcd5a47ce37bc632b08c5c2e117ae0af6bddc05d82138ecd7245a28f6a6c",
"x": "fX5C9uHGorCmQCuwcraTTxZPpbxUh3elPZt5uX3OvqA",
"kid": "cp-2027-01"
},
"cp_old": {
"seed_hex": "e73ef72607d47b301ffc20f914f2f5d1cfec889956733a638f36b39ca44ec4cd",
"x": "sWrWvn1edtMmmO_FjDMCyue2t9Y_tf_7eFSYrI1vBr0",
"kid": "cp-2026-10"
},
"cp_rogue": {
"seed_hex": "b280a5927ce64182115f2e7a834fdcc3ff8dd4b7ea7eebc42ada800f19f1c131",
"x": "87rSOETX2eSX4bDGWUAR3iszXDTEvANkf3j8Z8VYoz8",
"kid": "cp-rogue"
}
},
"nodes": {
"node_t1": {
"seed_hex": "8f4a1ad60c0a8ce22a04600cd1bbffbabe85577d7bc07fe3c80338dba5d2c1e0",
"x": "Pur4yosK8_cuGyraj4T-gI_Zxkb1wYrL1wNJqsK-50Q",
"kid": "node-t1-1",
"node": "t1"
},
"node_t2": {
"seed_hex": "4b386a098e6dbd89ef018880fea627c341aa4f3b854c2afe72db6480a805bbc2",
"x": "l73wsYm8_aBD6pz-Aj-_tgdOcgyaZYreC9wIP-a-UcA",
"kid": "node-t2-1",
"node": "t2"
},
"node_rogue": {
"seed_hex": "be1178b7dceb0556709593be15c92ef1fb25c9d5b02ef777bf2f734d232ae817",
"x": "F3Uod5FB1Ywy-iIFhRc3s5Zgy6NUzuTJJ7UsmhnMNKI",
"kid": "node-t1-0",
"node": "t1"
}
},
"devices": {
"a": {
"seed_hex": "911c4d06e2daa6ff6f83f5751ef7d5ba350d83ced323d1bebf3ee38de5ffe632",
"x": "j72RSCHD2lGtEh2gHTZ_LFhKtTUdArwkm2VDEVAPtPA",
"device_id": "b290d929d06d7dd6c9a98a5b8a5276aa",
"label": "psrhz6n5wpzbm6yfkngo",
"host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test"
},
"b": {
"seed_hex": "96bee129e7a2b918b542347dde37e22f0257bb26ea709c786aff7395eb112e5b",
"x": "7V0C8fhvAvQOcs_38WQLHcVtdbs8D1eOU6FDB8ocvAQ",
"device_id": "ac8fb81efc2ce011a6ded256c40a1d3a",
"label": "zoplfhior7zh6nwqxs6q",
"host": "zoplfhior7zh6nwqxs6q.devices.all-e2e.test"
}
},
"jwks": {
"cp": {
"keys": [
{
"kty": "OKP",
"crv": "Ed25519",
"x": "fX5C9uHGorCmQCuwcraTTxZPpbxUh3elPZt5uX3OvqA",
"kid": "cp-2027-01",
"alg": "EdDSA",
"use": "sig",
"key_ops": [
"verify"
]
},
{
"kty": "OKP",
"crv": "Ed25519",
"x": "sWrWvn1edtMmmO_FjDMCyue2t9Y_tf_7eFSYrI1vBr0",
"kid": "cp-2026-10",
"alg": "EdDSA",
"use": "sig",
"key_ops": [
"verify"
]
}
]
}
},
"node_keys": {
"live": {
"keys": [
{
"kid": "node-t1-1",
"node": "t1",
"kty": "OKP",
"crv": "Ed25519",
"x": "Pur4yosK8_cuGyraj4T-gI_Zxkb1wYrL1wNJqsK-50Q"
},
{
"kid": "node-t2-1",
"node": "t2",
"kty": "OKP",
"crv": "Ed25519",
"x": "l73wsYm8_aBD6pz-Aj-_tgdOcgyaZYreC9wIP-a-UcA"
}
]
}
},
"surface": {
"secret": "1E0RNbrxcukCk0rTUV8cQJ80BKH4PeJ228NtNdk9v7Q",
"pid": 4242,
"port": 45123,
"host": "127.0.0.1"
}
}
files/contracts/vectors/labels.json 55 lines · 4.6 KB
{
"labels": [
{"id": "label.ok.0", "value": "psrhz6n5wpzbm6yfkngo", "ok": true},
{"id": "label.ok.1", "value": "zoplfhior7zh6nwqxs6q", "ok": true},
{"id": "label.ok.2", "value": "aaaaaaaaaaaaaaaaaaaa", "ok": true},
{"id": "label.ok.3", "value": "zzzzzzzzzzzzzzzzzzzz", "ok": true},
{"id": "label.ok.4", "value": "22222222222222222222", "ok": true},
{"id": "label.ok.5", "value": "77777777777777777777", "ok": true},
{"id": "label.ok.6", "value": "abcdefghijklmnopqrst", "ok": true},
{"id": "label.ok.7", "value": "234567abcdefghijklmn", "ok": true},
{"id": "label.bad.nineteen", "value": "aaaaaaaaaaaaaaaaaaa", "ok": false},
{"id": "label.bad.twenty_one", "value": "aaaaaaaaaaaaaaaaaaaaa", "ok": false},
{"id": "label.bad.empty", "value": "", "ok": false},
{"id": "label.bad.uppercase", "value": "Aaaaaaaaaaaaaaaaaaaa", "ok": false},
{"id": "label.bad.underscore", "value": "aaaaaaaaaaaaaaaaaaa_", "ok": false},
{"id": "label.bad.dash", "value": "aaaaaaaaaaaaaaaaaaa-", "ok": false},
{"id": "label.bad.digit_zero", "value": "0aaaaaaaaaaaaaaaaaaa", "ok": false},
{"id": "label.bad.digit_one", "value": "1aaaaaaaaaaaaaaaaaaa", "ok": false},
{"id": "label.bad.digit_eight", "value": "8aaaaaaaaaaaaaaaaaaa", "ok": false},
{"id": "label.bad.digit_nine", "value": "9aaaaaaaaaaaaaaaaaaa", "ok": false},
{"id": "label.bad.trailing_newline", "value": "aaaaaaaaaaaaaaaaaaaa\n", "ok": false},
{"id": "label.bad.leading_space", "value": " aaaaaaaaaaaaaaaaaaa", "ok": false},
{"id": "label.bad.trailing_dot", "value": "aaaaaaaaaaaaaaaaaaa.", "ok": false},
{"id": "label.bad.cyrillic_a", "value": "\u0430aaaaaaaaaaaaaaaaaaa", "ok": false},
{"id": "label.bad.reserved_name", "value": "t1", "ok": false},
{"id": "label.bad.dot_inside", "value": "aaaaaaaaaa.aaaaaaaaa", "ok": false},
{"id": "label.bad.sixty_four", "value": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "ok": false}
],
"fqdns": [
{"id": "fqdn.ok.0", "value": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "ok": true},
{"id": "fqdn.ok.1", "value": "zoplfhior7zh6nwqxs6q.devices.all-e2e.test", "ok": true},
{"id": "fqdn.ok.2", "value": "psrhz6n5wpzbm6yfkngo.example.com", "ok": true},
{"id": "fqdn.ok.3", "value": "psrhz6n5wpzbm6yfkngo.a.b.example.org", "ok": true},
{"id": "fqdn.bad.uppercase_host", "value": "PSRHZ6N5WPZBM6YFKNGO.DEVICES.ALL-E2E.TEST", "ok": false},
{"id": "fqdn.bad.uppercase_label", "value": "PSRHZ6N5WPZBM6YFKNGO.devices.all-e2e.test", "ok": false},
{"id": "fqdn.bad.underscore_label", "value": "aaaaaaaaaaaaaaaaaaa_.devices.all-e2e.test", "ok": false},
{"id": "fqdn.bad.trailing_dot", "value": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test.", "ok": false},
{"id": "fqdn.bad.bare_label", "value": "psrhz6n5wpzbm6yfkngo", "ok": false},
{"id": "fqdn.bad.label_only_tld", "value": "psrhz6n5wpzbm6yfkngo.test1", "ok": false},
{"id": "fqdn.bad.wildcard", "value": "*.devices.all-e2e.test", "ok": false},
{"id": "fqdn.bad.empty_label", "value": "psrhz6n5wpzbm6yfkngo.devices..test", "ok": false},
{"id": "fqdn.bad.two_label_prefix", "value": "x.psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "ok": false},
{"id": "fqdn.bad.short_label", "value": "abc.devices.all-e2e.test", "ok": false},
{"id": "fqdn.bad.underscore_zone", "value": "psrhz6n5wpzbm6yfkngo.de_vices.test", "ok": false},
{"id": "fqdn.bad.leading_hyphen_zone", "value": "psrhz6n5wpzbm6yfkngo.-devices.test", "ok": false},
{"id": "fqdn.bad.ipv4", "value": "192.0.2.10", "ok": false},
{"id": "fqdn.bad.too_long", "value": "psrhz6n5wpzbm6yfkngo.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.test", "ok": false},
{"id": "fqdn.bad.trailing_newline", "value": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test\n", "ok": false},
{"id": "fqdn.bad.port", "value": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test:443", "ok": false},
{"id": "fqdn.bad.url", "value": "https://psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "ok": false},
{"id": "fqdn.zone.listed", "value": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "zones": ["devices.all-e2e.test", "other.test"], "ok": true},
{"id": "fqdn.zone.unlisted", "value": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "zones": ["other.test"], "ok": false},
{"id": "fqdn.zone.suffix_trick", "value": "psrhz6n5wpzbm6yfkngo.evil-devices.all-e2e.test", "zones": ["devices.all-e2e.test"], "ok": false}
]
}
files/contracts/vectors/lines.json 155 lines · 113.8 KB
{
"lines": [
{"id": "line.ok.hello", "dir": "d2n", "line": "HELLO 2 eyJhbGciOiJFZERTQSIsImtpZCI6ImNwLTIwMjctMDEiLCJ0eXAiOiJkZXZpY2Utc2Vzc2lvbiJ9.eyJ0eXAiOiJkZXZpY2Utc2Vzc2lvbiIsInZlciI6MiwiaXNzIjoiaHR0cHM6Ly9hcGkuYWxsLW1jcC5meWkiLCJhdWQiOiJhbGwtdHVubmVsIiwic3ViIjoiYjI5MGQ5MjlkMDZkN2RkNmM5YTk4YTViOGE1Mjc2YWEiLCJpYXQiOjE3OTg3NjE3MDAsImV4cCI6MTc5ODc2MjAwMCwianRpIjoicXdIVndyZ1lTS2xTbC1VNjdieHNCNiIsImdlbmVyYXRpb24iOjEsImNuZiI6eyJqd2siOnsia3R5IjoiT0tQIiwiY3J2IjoiRWQyNTUxOSIsIngiOiJqNzJSU0NIRDJsR3RFaDJnSFRaX0xGaEt0VFVkQXJ3a20yVkRFVkFQdFBBIn19fQ.Rt1VAagkgfgrgkf7sXVHxIQwwzwxSRG_NMdnp0h3K-RgXzDdMWDQ4Guv81HMJsXgzducQagBwywsU1p4GEW5AQ", "ok": true, "parsed": {"verb": "HELLO", "version": "2", "session": "eyJhbGciOiJFZERTQSIsImtpZCI6ImNwLTIwMjctMDEiLCJ0eXAiOiJkZXZpY2Utc2Vzc2lvbiJ9.eyJ0eXAiOiJkZXZpY2Utc2Vzc2lvbiIsInZlciI6MiwiaXNzIjoiaHR0cHM6Ly9hcGkuYWxsLW1jcC5meWkiLCJhdWQiOiJhbGwtdHVubmVsIiwic3ViIjoiYjI5MGQ5MjlkMDZkN2RkNmM5YTk4YTViOGE1Mjc2YWEiLCJpYXQiOjE3OTg3NjE3MDAsImV4cCI6MTc5ODc2MjAwMCwianRpIjoicXdIVndyZ1lTS2xTbC1VNjdieHNCNiIsImdlbmVyYXRpb24iOjEsImNuZiI6eyJqd2siOnsia3R5IjoiT0tQIiwiY3J2IjoiRWQyNTUxOSIsIngiOiJqNzJSU0NIRDJsR3RFaDJnSFRaX0xGaEt0VFVkQXJ3a20yVkRFVkFQdFBBIn19fQ.Rt1VAagkgfgrgkf7sXVHxIQwwzwxSRG_NMdnp0h3K-RgXzDdMWDQ4Guv81HMJsXgzducQagBwywsU1p4GEW5AQ"}, "canonical": true},
{"id": "line.ok.proof", "dir": "d2n", "line": "PROOF eyJhbGciOiJFZERTQSIsInR5cCI6ImRwb3Arand0IiwiandrIjp7Imt0eSI6Ik9LUCIsImNydiI6IkVkMjU1MTkiLCJ4IjoiajcyUlNDSEQybEd0RWgyZ0hUWl9MRmhLdFRVZEFyd2ttMlZERVZBUHRQQSJ9fQ.eyJzdWIiOiJiMjkwZDkyOWQwNmQ3ZGQ2YzlhOThhNWI4YTUyNzZhYSIsImF1ZCI6ImFsbC10dW5uZWwiLCJodG0iOiJHRVQiLCJodHUiOiJodHRwczovL3QxLm5vZGVzLmFsbC1lMmUudGVzdC9hbGwtdHVubmVsLzIiLCJpYXQiOjE3OTg3NjE3MDAsImp0aSI6Ik1lU3Y2SVFGdnJyTlFEbWlPR2hzMVkiLCJhdGgiOiJuSmhma3lqVlRHMDBOU0Q2aTNOc2d0TUdrZVRIUWtJMUJqTHpsYXVKSFFrIiwibm9uY2UiOiJQaHJrTE5kRmhlODd6UUpoTVpFakFCIn0.B6KTGID_KGdC6naKMPPs-8jRkZAAjDpKIubsmbIdwg_P5HVpQeoUMENvghhEVfikeJR-DwNeSdRqSs9LnQmWAg", "ok": true, "parsed": {"verb": "PROOF", "proof": "eyJhbGciOiJFZERTQSIsInR5cCI6ImRwb3Arand0IiwiandrIjp7Imt0eSI6Ik9LUCIsImNydiI6IkVkMjU1MTkiLCJ4IjoiajcyUlNDSEQybEd0RWgyZ0hUWl9MRmhLdFRVZEFyd2ttMlZERVZBUHRQQSJ9fQ.eyJzdWIiOiJiMjkwZDkyOWQwNmQ3ZGQ2YzlhOThhNWI4YTUyNzZhYSIsImF1ZCI6ImFsbC10dW5uZWwiLCJodG0iOiJHRVQiLCJodHUiOiJodHRwczovL3QxLm5vZGVzLmFsbC1lMmUudGVzdC9hbGwtdHVubmVsLzIiLCJpYXQiOjE3OTg3NjE3MDAsImp0aSI6Ik1lU3Y2SVFGdnJyTlFEbWlPR2hzMVkiLCJhdGgiOiJuSmhma3lqVlRHMDBOU0Q2aTNOc2d0TUdrZVRIUWtJMUJqTHpsYXVKSFFrIiwibm9uY2UiOiJQaHJrTE5kRmhlODd6UUpoTVpFakFCIn0.B6KTGID_KGdC6naKMPPs-8jRkZAAjDpKIubsmbIdwg_P5HVpQeoUMENvghhEVfikeJR-DwNeSdRqSs9LnQmWAg"}, "canonical": true},
{"id": "line.ok.renew", "dir": "d2n", "line": "RENEW eyJhbGciOiJFZERTQSIsImtpZCI6ImNwLTIwMjctMDEiLCJ0eXAiOiJkZXZpY2Utc2Vzc2lvbiJ9.eyJ0eXAiOiJkZXZpY2Utc2Vzc2lvbiIsInZlciI6MiwiaXNzIjoiaHR0cHM6Ly9hcGkuYWxsLW1jcC5meWkiLCJhdWQiOiJhbGwtdHVubmVsIiwic3ViIjoiYjI5MGQ5MjlkMDZkN2RkNmM5YTk4YTViOGE1Mjc2YWEiLCJpYXQiOjE3OTg3NjE3MDAsImV4cCI6MTc5ODc2MjAwMCwianRpIjoicXdIVndyZ1lTS2xTbC1VNjdieHNCNiIsImdlbmVyYXRpb24iOjEsImNuZiI6eyJqd2siOnsia3R5IjoiT0tQIiwiY3J2IjoiRWQyNTUxOSIsIngiOiJqNzJSU0NIRDJsR3RFaDJnSFRaX0xGaEt0VFVkQXJ3a20yVkRFVkFQdFBBIn19fQ.Rt1VAagkgfgrgkf7sXVHxIQwwzwxSRG_NMdnp0h3K-RgXzDdMWDQ4Guv81HMJsXgzducQagBwywsU1p4GEW5AQ", "ok": true, "parsed": {"verb": "RENEW", "session": "eyJhbGciOiJFZERTQSIsImtpZCI6ImNwLTIwMjctMDEiLCJ0eXAiOiJkZXZpY2Utc2Vzc2lvbiJ9.eyJ0eXAiOiJkZXZpY2Utc2Vzc2lvbiIsInZlciI6MiwiaXNzIjoiaHR0cHM6Ly9hcGkuYWxsLW1jcC5meWkiLCJhdWQiOiJhbGwtdHVubmVsIiwic3ViIjoiYjI5MGQ5MjlkMDZkN2RkNmM5YTk4YTViOGE1Mjc2YWEiLCJpYXQiOjE3OTg3NjE3MDAsImV4cCI6MTc5ODc2MjAwMCwianRpIjoicXdIVndyZ1lTS2xTbC1VNjdieHNCNiIsImdlbmVyYXRpb24iOjEsImNuZiI6eyJqd2siOnsia3R5IjoiT0tQIiwiY3J2IjoiRWQyNTUxOSIsIngiOiJqNzJSU0NIRDJsR3RFaDJnSFRaX0xGaEt0VFVkQXJ3a20yVkRFVkFQdFBBIn19fQ.Rt1VAagkgfgrgkf7sXVHxIQwwzwxSRG_NMdnp0h3K-RgXzDdMWDQ4Guv81HMJsXgzducQagBwywsU1p4GEW5AQ"}, "canonical": true},
{"id": "line.ok.ping", "dir": "d2n", "line": "PING", "ok": true, "parsed": {"verb": "PING"}, "canonical": true},
{"id": "line.ok.hint_one", "dir": "d2n", "line": "HINT 203.0.113.7", "ok": true, "parsed": {"verb": "HINT", "addresses": ["203.0.113.7"]}, "canonical": true},
{"id": "line.ok.hint_mixed", "dir": "d2n", "line": "HINT 203.0.113.7 2001:db8::1 198.51.100.200 2001:db8:85a3::8a2e:370:7334", "ok": true, "parsed": {"verb": "HINT", "addresses": ["203.0.113.7", "2001:db8::1", "198.51.100.200", "2001:db8:85a3::8a2e:370:7334"]}, "canonical": true},
{"id": "line.ok.hint_max", "dir": "d2n", "line": "HINT 192.0.2.1 192.0.2.2 192.0.2.3 192.0.2.4 192.0.2.5 192.0.2.6 192.0.2.7 192.0.2.8 192.0.2.9 192.0.2.10 192.0.2.11 192.0.2.12 192.0.2.13 192.0.2.14 192.0.2.15 192.0.2.16", "ok": true, "parsed": {"verb": "HINT", "addresses": ["192.0.2.1", "192.0.2.2", "192.0.2.3", "192.0.2.4", "192.0.2.5", "192.0.2.6", "192.0.2.7", "192.0.2.8", "192.0.2.9", "192.0.2.10", "192.0.2.11", "192.0.2.12", "192.0.2.13", "192.0.2.14", "192.0.2.15", "192.0.2.16"]}, "canonical": true},
{"id": "line.ok.close_bare", "dir": "d2n", "line": "CLOSE", "ok": true, "parsed": {"verb": "CLOSE"}, "canonical": true},
{"id": "line.ok.close_reason", "dir": "d2n", "line": "CLOSE shutdown", "ok": true, "parsed": {"verb": "CLOSE", "reason": "shutdown"}, "canonical": true},
{"id": "line.ok.attach", "dir": "d2n", "line": "ATTACH 18e1c58e65750259573cfafb3af1ebcb MLgSPl2_k3Hok9SAoC3XYiB3e6IM447w7IOlRffiAmg", "ok": true, "parsed": {"verb": "ATTACH", "stream_id": "18e1c58e65750259573cfafb3af1ebcb", "secret": "MLgSPl2_k3Hok9SAoC3XYiB3e6IM447w7IOlRffiAmg"}, "canonical": true},
{"id": "line.ok.challenge", "dir": "n2d", "line": "CHALLENGE PN_0J0TsTUh5lIyBmCO7dG", "ok": true, "parsed": {"verb": "CHALLENGE", "nonce": "PN_0J0TsTUh5lIyBmCO7dG"}, "canonical": true},
{"id": "line.ok.ok", "dir": "n2d", "line": "OK {\"host\":\"psrhz6n5wpzbm6yfkngo.devices.all-e2e.test\",\"node\":\"t1\",\"policy\":{\"dead_peer_s\":45,\"ping_interval_s\":20,\"renew_before_valid_until_s\":300,\"stream_idle_s\":1800,\"stream_max_age_s\":14400},\"server_time\":1798761700,\"status\":\"active\",\"v\":2,\"valid_until\":1798776100}", "ok": true, "parsed": {"verb": "OK", "body": {"v": 2, "node": "t1", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "status": "active", "valid_until": 1798776100, "server_time": 1798761700, "policy": {"ping_interval_s": 20, "dead_peer_s": 45, "stream_idle_s": 1800, "stream_max_age_s": 14400, "renew_before_valid_until_s": 300}}}, "canonical": true},
{"id": "line.ok.ok_not_canonical_still_parses", "dir": "n2d", "ok": true, "parsed": {"verb": "OK", "body": {"v": 2, "node": "t1", "host": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test", "status": "active", "valid_until": 1798776100, "server_time": 1798761700, "policy": {"ping_interval_s": 20, "dead_peer_s": 45, "stream_idle_s": 1800, "stream_max_age_s": 14400, "renew_before_valid_until_s": 300}}}, "canonical": false, "line": "OK {\"v\": 2, \"node\": \"t1\", \"host\": \"psrhz6n5wpzbm6yfkngo.devices.all-e2e.test\", \"status\": \"active\", \"valid_until\": 1798776100, \"server_time\": 1798761700, \"policy\": {\"ping_interval_s\": 20, \"dead_peer_s\": 45, \"stream_idle_s\": 1800, \"stream_max_age_s\": 14400, \"renew_before_valid_until_s\": 300}}"},
{"id": "line.ok.err", "dir": "n2d", "line": "ERR auth", "ok": true, "parsed": {"verb": "ERR", "code": "auth"}, "canonical": true},
{"id": "line.ok.err_clock", "dir": "n2d", "line": "ERR clock_skew", "ok": true, "parsed": {"verb": "ERR", "code": "clock_skew"}, "canonical": true},
{"id": "line.ok.open_openai", "dir": "n2d", "line": "OPEN 18e1c58e65750259573cfafb3af1ebcb MLgSPl2_k3Hok9SAoC3XYiB3e6IM447w7IOlRffiAmg 203.0.113.9 openai", "ok": true, "parsed": {"verb": "OPEN", "stream_id": "18e1c58e65750259573cfafb3af1ebcb", "secret": "MLgSPl2_k3Hok9SAoC3XYiB3e6IM447w7IOlRffiAmg", "peer_ip": "203.0.113.9", "class": "openai"}, "canonical": true},
{"id": "line.ok.open_hint", "dir": "n2d", "line": "OPEN 18e1c58e65750259573cfafb3af1ebcb MLgSPl2_k3Hok9SAoC3XYiB3e6IM447w7IOlRffiAmg 203.0.113.9 hint", "ok": true, "parsed": {"verb": "OPEN", "stream_id": "18e1c58e65750259573cfafb3af1ebcb", "secret": "MLgSPl2_k3Hok9SAoC3XYiB3e6IM447w7IOlRffiAmg", "peer_ip": "203.0.113.9", "class": "hint"}, "canonical": true},
{"id": "line.ok.open_anon", "dir": "n2d", "line": "OPEN 18e1c58e65750259573cfafb3af1ebcb MLgSPl2_k3Hok9SAoC3XYiB3e6IM447w7IOlRffiAmg 203.0.113.9 anon", "ok": true, "parsed": {"verb": "OPEN", "stream_id": "18e1c58e65750259573cfafb3af1ebcb", "secret": "MLgSPl2_k3Hok9SAoC3XYiB3e6IM447w7IOlRffiAmg", "peer_ip": "203.0.113.9", "class": "anon"}, "canonical": true},
{"id": "line.ok.open_ipv6", "dir": "n2d", "line": "OPEN 18e1c58e65750259573cfafb3af1ebcb MLgSPl2_k3Hok9SAoC3XYiB3e6IM447w7IOlRffiAmg 2001:db8::7 openai", "ok": true, "parsed": {"verb": "OPEN", "stream_id": "18e1c58e65750259573cfafb3af1ebcb", "secret": "MLgSPl2_k3Hok9SAoC3XYiB3e6IM447w7IOlRffiAmg", "peer_ip": "2001:db8::7", "class": "openai"}, "canonical": true},
{"id": "line.ok.pong", "dir": "n2d", "line": "PONG", "ok": true, "parsed": {"verb": "PONG"}, "canonical": true},
{"id": "line.ok.statement", "dir": "n2d", "line": "STATEMENT eyJhbGciOiJFZERTQSIsImtpZCI6ImNwLTIwMjctMDEiLCJ0eXAiOiJkZXZpY2UtYWNjZXNzIn0.eyJ0eXAiOiJkZXZpY2UtYWNjZXNzIiwidmVyIjoxLCJpc3MiOiJodHRwczovL2FwaS5hbGwtbWNwLmZ5aSIsImF1ZCI6ImFsbC1kZXZpY2UtYWNjZXNzIiwic3ViIjoiYjI5MGQ5MjlkMDZkN2RkNmM5YTk4YTViOGE1Mjc2YWEiLCJpYXQiOjE3OTg3NjE3MDAsImV4cCI6MTc5ODc3NjEwMCwianRpIjoiT2ZUaENfWHB1SXFxYnNCNEQwV2FvUyIsIm1vZGUiOiJhY3RpdmUiLCJzZXEiOjF9.3UtIM8YqoOLuB0tV5r70K7HGGJ5yw7JB0TyhNmvPRQSwmsUS96zD71szA0JxnqgQOD350370QeeOcxg5PQuGDw", "ok": true, "parsed": {"verb": "STATEMENT", "statement": "eyJhbGciOiJFZERTQSIsImtpZCI6ImNwLTIwMjctMDEiLCJ0eXAiOiJkZXZpY2UtYWNjZXNzIn0.eyJ0eXAiOiJkZXZpY2UtYWNjZXNzIiwidmVyIjoxLCJpc3MiOiJodHRwczovL2FwaS5hbGwtbWNwLmZ5aSIsImF1ZCI6ImFsbC1kZXZpY2UtYWNjZXNzIiwic3ViIjoiYjI5MGQ5MjlkMDZkN2RkNmM5YTk4YTViOGE1Mjc2YWEiLCJpYXQiOjE3OTg3NjE3MDAsImV4cCI6MTc5ODc3NjEwMCwianRpIjoiT2ZUaENfWHB1SXFxYnNCNEQwV2FvUyIsIm1vZGUiOiJhY3RpdmUiLCJzZXEiOjF9.3UtIM8YqoOLuB0tV5r70K7HGGJ5yw7JB0TyhNmvPRQSwmsUS96zD71szA0JxnqgQOD350370QeeOcxg5PQuGDw"}, "canonical": true},
{"id": "line.ok.drain", "dir": "n2d", "line": "DRAIN 1200", "ok": true, "parsed": {"verb": "DRAIN", "quiet_s": "1200"}, "canonical": true},
{"id": "line.ok.close_n2d", "dir": "n2d", "line": "CLOSE expired", "ok": true, "parsed": {"verb": "CLOSE", "reason": "expired"}, "canonical": true},
{"id": "line.bad.hello_v1", "dir": "d2n", "line": "HELLO 1 eyJhbGciOiJFZERTQSIsImtpZCI6ImNwLTIwMjctMDEiLCJ0eXAiOiJkZXZpY2Utc2Vzc2lvbiJ9.eyJ0eXAiOiJkZXZpY2Utc2Vzc2lvbiIsInZlciI6MiwiaXNzIjoiaHR0cHM6Ly9hcGkuYWxsLW1jcC5meWkiLCJhdWQiOiJhbGwtdHVubmVsIiwic3ViIjoiYjI5MGQ5MjlkMDZkN2RkNmM5YTk4YTViOGE1Mjc2YWEiLCJpYXQiOjE3OTg3NjE3MDAsImV4cCI6MTc5ODc2MjAwMCwianRpIjoicXdIVndyZ1lTS2xTbC1VNjdieHNCNiIsImdlbmVyYXRpb24iOjEsImNuZiI6eyJqd2siOnsia3R5IjoiT0tQIiwiY3J2IjoiRWQyNTUxOSIsIngiOiJqNzJSU0NIRDJsR3RFaDJnSFRaX0xGaEt0VFVkQXJ3a20yVkRFVkFQdFBBIn19fQ.Rt1VAagkgfgrgkf7sXVHxIQwwzwxSRG_NMdnp0h3K-RgXzDdMWDQ4Guv81HMJsXgzducQagBwywsU1p4GEW5AQ", "ok": false, "error": "unsupported_version"},
{"id": "line.bad.hello_v3", "dir": "d2n", "line": "HELLO 3 eyJhbGciOiJFZERTQSIsImtpZCI6ImNwLTIwMjctMDEiLCJ0eXAiOiJkZXZpY2Utc2Vzc2lvbiJ9.eyJ0eXAiOiJkZXZpY2Utc2Vzc2lvbiIsInZlciI6MiwiaXNzIjoiaHR0cHM6Ly9hcGkuYWxsLW1jcC5meWkiLCJhdWQiOiJhbGwtdHVubmVsIiwic3ViIjoiYjI5MGQ5MjlkMDZkN2RkNmM5YTk4YTViOGE1Mjc2YWEiLCJpYXQiOjE3OTg3NjE3MDAsImV4cCI6MTc5ODc2MjAwMCwianRpIjoicXdIVndyZ1lTS2xTbC1VNjdieHNCNiIsImdlbmVyYXRpb24iOjEsImNuZiI6eyJqd2siOnsia3R5IjoiT0tQIiwiY3J2IjoiRWQyNTUxOSIsIngiOiJqNzJSU0NIRDJsR3RFaDJnSFRaX0xGaEt0VFVkQXJ3a20yVkRFVkFQdFBBIn19fQ.Rt1VAagkgfgrgkf7sXVHxIQwwzwxSRG_NMdnp0h3K-RgXzDdMWDQ4Guv81HMJsXgzducQagBwywsU1p4GEW5AQ", "ok": false, "error": "unsupported_version"},
{"id": "line.bad.hello_version_text", "dir": "d2n", "line": "HELLO two eyJhbGciOiJFZERTQSIsImtpZCI6ImNwLTIwMjctMDEiLCJ0eXAiOiJkZXZpY2Utc2Vzc2lvbiJ9.eyJ0eXAiOiJkZXZpY2Utc2Vzc2lvbiIsInZlciI6MiwiaXNzIjoiaHR0cHM6Ly9hcGkuYWxsLW1jcC5meWkiLCJhdWQiOiJhbGwtdHVubmVsIiwic3ViIjoiYjI5MGQ5MjlkMDZkN2RkNmM5YTk4YTViOGE1Mjc2YWEiLCJpYXQiOjE3OTg3NjE3MDAsImV4cCI6MTc5ODc2MjAwMCwianRpIjoicXdIVndyZ1lTS2xTbC1VNjdieHNCNiIsImdlbmVyYXRpb24iOjEsImNuZiI6eyJqd2siOnsia3R5IjoiT0tQIiwiY3J2IjoiRWQyNTUxOSIsIngiOiJqNzJSU0NIRDJsR3RFaDJnSFRaX0xGaEt0VFVkQXJ3a20yVkRFVkFQdFBBIn19fQ.Rt1VAagkgfgrgkf7sXVHxIQwwzwxSRG_NMdnp0h3K-RgXzDdMWDQ4Guv81HMJsXgzducQagBwywsU1p4GEW5AQ", "ok": false, "error": "bad_command"},
{"id": "line.bad.hello_missing_session", "dir": "d2n", "line": "HELLO 2", "ok": false, "error": "bad_command"},
{"id": "line.bad.hello_not_a_jwt", "dir": "d2n", "line": "HELLO 2 notatoken", "ok": false, "error": "bad_hello"},
{"id": "line.bad.hello_oversized_token", "dir": "d2n", "line": "HELLO 2 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.b.cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "ok": false, "error": "bad_hello"},
{"id": "line.bad.hello_line_too_long", "dir": "d2n", "line": "HELLO 2 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "ok": false, "error": "line_too_long"},
{"id": "line.bad.hello_extra_field", "dir": "d2n", "line": "HELLO 2 eyJhbGciOiJFZERTQSIsImtpZCI6ImNwLTIwMjctMDEiLCJ0eXAiOiJkZXZpY2Utc2Vzc2lvbiJ9.eyJ0eXAiOiJkZXZpY2Utc2Vzc2lvbiIsInZlciI6MiwiaXNzIjoiaHR0cHM6Ly9hcGkuYWxsLW1jcC5meWkiLCJhdWQiOiJhbGwtdHVubmVsIiwic3ViIjoiYjI5MGQ5MjlkMDZkN2RkNmM5YTk4YTViOGE1Mjc2YWEiLCJpYXQiOjE3OTg3NjE3MDAsImV4cCI6MTc5ODc2MjAwMCwianRpIjoicXdIVndyZ1lTS2xTbC1VNjdieHNCNiIsImdlbmVyYXRpb24iOjEsImNuZiI6eyJqd2siOnsia3R5IjoiT0tQIiwiY3J2IjoiRWQyNTUxOSIsIngiOiJqNzJSU0NIRDJsR3RFaDJnSFRaX0xGaEt0VFVkQXJ3a20yVkRFVkFQdFBBIn19fQ.Rt1VAagkgfgrgkf7sXVHxIQwwzwxSRG_NMdnp0h3K-RgXzDdMWDQ4Guv81HMJsXgzducQagBwywsU1p4GEW5AQ extra", "ok": false, "error": "bad_command"},
{"id": "line.bad.proof_garbage", "dir": "d2n", "line": "PROOF %%%", "ok": false, "error": "bad_proof"},
{"id": "line.bad.renew_garbage", "dir": "d2n", "line": "RENEW a.b", "ok": false, "error": "bad_hello"},
{"id": "line.bad.attach_short_id", "dir": "d2n", "line": "ATTACH 18e1c58e65750259573cfafb3af1ebc MLgSPl2_k3Hok9SAoC3XYiB3e6IM447w7IOlRffiAmg", "ok": false, "error": "bad_attach"},
{"id": "line.bad.attach_upper_id", "dir": "d2n", "line": "ATTACH 18E1C58E65750259573CFAFB3AF1EBCB MLgSPl2_k3Hok9SAoC3XYiB3e6IM447w7IOlRffiAmg", "ok": false, "error": "bad_attach"},
{"id": "line.bad.attach_short_secret", "dir": "d2n", "line": "ATTACH 18e1c58e65750259573cfafb3af1ebcb MLgSPl2_k3Hok9SAoC3XYiB3e6IM447w7IOlRffiAm", "ok": false, "error": "bad_attach"},
{"id": "line.bad.attach_missing_secret", "dir": "d2n", "line": "ATTACH 18e1c58e65750259573cfafb3af1ebcb", "ok": false, "error": "bad_command"},
{"id": "line.bad.open_oversized", "dir": "n2d", "line": "OPEN 18e1c58e65750259573cfafb3af1ebcb MLgSPl2_k3Hok9SAoC3XYiB3e6IM447w7IOlRffiAmgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA 203.0.113.9 openai", "ok": false, "error": "line_too_long"},
{"id": "line.bad.open_truncated", "dir": "n2d", "line": "OPEN 18e1c58e65750259573cfafb3af1ebcb MLgSPl2_k3Hok9SAoC3X", "ok": false, "error": "bad_command"},
{"id": "line.bad.open_three_fields", "dir": "n2d", "line": "OPEN 18e1c58e65750259573cfafb3af1ebcb MLgSPl2_k3Hok9SAoC3XYiB3e6IM447w7IOlRffiAmg 203.0.113.9", "ok": false, "error": "bad_command"},
{"id": "line.bad.open_class", "dir": "n2d", "line": "OPEN 18e1c58e65750259573cfafb3af1ebcb MLgSPl2_k3Hok9SAoC3XYiB3e6IM447w7IOlRffiAmg 203.0.113.9 evil", "ok": false, "error": "bad_command"},
{"id": "line.bad.open_upper_ip", "dir": "n2d", "line": "OPEN 18e1c58e65750259573cfafb3af1ebcb MLgSPl2_k3Hok9SAoC3XYiB3e6IM447w7IOlRffiAmg 2001:DB8::7 openai", "ok": false, "error": "bad_command"},
{"id": "line.bad.open_zone_ip", "dir": "n2d", "line": "OPEN 18e1c58e65750259573cfafb3af1ebcb MLgSPl2_k3Hok9SAoC3XYiB3e6IM447w7IOlRffiAmg fe80::1%en0 openai", "ok": false, "error": "bad_command"},
{"id": "line.bad.open_leading_zero_ip", "dir": "n2d", "line": "OPEN 18e1c58e65750259573cfafb3af1ebcb MLgSPl2_k3Hok9SAoC3XYiB3e6IM447w7IOlRffiAmg 203.0.113.09 openai", "ok": false, "error": "bad_command"},
{"id": "line.bad.open_bracket_ip", "dir": "n2d", "line": "OPEN 18e1c58e65750259573cfafb3af1ebcb MLgSPl2_k3Hok9SAoC3XYiB3e6IM447w7IOlRffiAmg [2001:db8::7] openai", "ok": false, "error": "bad_command"},
{"id": "line.bad.open_hostname", "dir": "n2d", "line": "OPEN 18e1c58e65750259573cfafb3af1ebcb MLgSPl2_k3Hok9SAoC3XYiB3e6IM447w7IOlRffiAmg example.com openai", "ok": false, "error": "bad_command"},
{"id": "line.bad.open_wrong_direction", "dir": "d2n", "line": "OPEN 18e1c58e65750259573cfafb3af1ebcb MLgSPl2_k3Hok9SAoC3XYiB3e6IM447w7IOlRffiAmg 203.0.113.9 openai", "ok": false, "error": "bad_command"},
{"id": "line.bad.hello_wrong_direction", "dir": "n2d", "line": "HELLO 2 eyJhbGciOiJFZERTQSIsImtpZCI6ImNwLTIwMjctMDEiLCJ0eXAiOiJkZXZpY2Utc2Vzc2lvbiJ9.eyJ0eXAiOiJkZXZpY2Utc2Vzc2lvbiIsInZlciI6MiwiaXNzIjoiaHR0cHM6Ly9hcGkuYWxsLW1jcC5meWkiLCJhdWQiOiJhbGwtdHVubmVsIiwic3ViIjoiYjI5MGQ5MjlkMDZkN2RkNmM5YTk4YTViOGE1Mjc2YWEiLCJpYXQiOjE3OTg3NjE3MDAsImV4cCI6MTc5ODc2MjAwMCwianRpIjoicXdIVndyZ1lTS2xTbC1VNjdieHNCNiIsImdlbmVyYXRpb24iOjEsImNuZiI6eyJqd2siOnsia3R5IjoiT0tQIiwiY3J2IjoiRWQyNTUxOSIsIngiOiJqNzJSU0NIRDJsR3RFaDJnSFRaX0xGaEt0VFVkQXJ3a20yVkRFVkFQdFBBIn19fQ.Rt1VAagkgfgrgkf7sXVHxIQwwzwxSRG_NMdnp0h3K-RgXzDdMWDQ4Guv81HMJsXgzducQagBwywsU1p4GEW5AQ", "ok": false, "error": "bad_command"},
{"id": "line.bad.lowercase_verb", "dir": "d2n", "line": "ping", "ok": false, "error": "bad_command"},
{"id": "line.bad.unknown_verb", "dir": "d2n", "line": "WAVE", "ok": false, "error": "bad_command"},
{"id": "line.bad.trailing_space", "dir": "d2n", "line": "PING ", "ok": false, "error": "bad_command"},
{"id": "line.bad.leading_space", "dir": "d2n", "line": " PING", "ok": false, "error": "bad_command"},
{"id": "line.bad.double_space", "dir": "n2d", "line": "OPEN 18e1c58e65750259573cfafb3af1ebcb MLgSPl2_k3Hok9SAoC3XYiB3e6IM447w7IOlRffiAmg 203.0.113.9 openai", "ok": false, "error": "bad_command"},
{"id": "line.bad.carriage_return", "dir": "d2n", "line": "PING\r", "ok": false, "error": "bad_command"},
{"id": "line.bad.tab", "dir": "d2n", "line": "HINT\t203.0.113.7", "ok": false, "error": "bad_command"},
{"id": "line.bad.non_ascii", "dir": "d2n", "line": "HINT 203.0.113.\u0663", "ok": false, "error": "bad_command"},
{"id": "line.bad.empty", "dir": "d2n", "line": "", "ok": false, "error": "bad_command"},
{"id": "line.bad.ping_arg", "dir": "d2n", "line": "PING now", "ok": false, "error": "bad_command"},
{"id": "line.bad.pong_arg", "dir": "n2d", "line": "PONG now", "ok": false, "error": "bad_command"},
{"id": "line.bad.hint_none", "dir": "d2n", "line": "HINT", "ok": false, "error": "bad_command"},
{"id": "line.bad.hint_seventeen", "dir": "d2n", "line": "HINT 192.0.2.1 192.0.2.2 192.0.2.3 192.0.2.4 192.0.2.5 192.0.2.6 192.0.2.7 192.0.2.8 192.0.2.9 192.0.2.10 192.0.2.11 192.0.2.12 192.0.2.13 192.0.2.14 192.0.2.15 192.0.2.16 192.0.2.17", "ok": false, "error": "bad_command"},
{"id": "line.bad.hint_hostname", "dir": "d2n", "line": "HINT example.com", "ok": false, "error": "bad_command"},
{"id": "line.bad.close_reason", "dir": "d2n", "line": "CLOSE because", "ok": false, "error": "bad_command"},
{"id": "line.bad.close_two_reasons", "dir": "n2d", "line": "CLOSE expired revoked", "ok": false, "error": "bad_command"},
{"id": "line.bad.challenge_short", "dir": "n2d", "line": "CHALLENGE abc", "ok": false, "error": "bad_command"},
{"id": "line.bad.challenge_punctuation", "dir": "n2d", "line": "CHALLENGE aaaaaaaaaaaaaaaaaaaaa!", "ok": false, "error": "bad_command"},
{"id": "line.bad.err_unknown", "dir": "n2d", "line": "ERR something_new", "ok": false, "error": "bad_command"},
{"id": "line.bad.err_none", "dir": "n2d", "line": "ERR", "ok": false, "error": "bad_command"},
{"id": "line.bad.ok_not_json", "dir": "n2d", "line": "OK {nope", "ok": false, "error": "bad_command"},
{"id": "line.bad.ok_empty", "dir": "n2d", "line": "OK", "ok": false, "error": "bad_command"},
{"id": "line.bad.ok_array", "dir": "n2d", "line": "OK []", "ok": false, "error": "bad_command"},
{"id": "line.bad.ok_extra_key", "dir": "n2d", "line": "OK {\"extra\":1,\"host\":\"psrhz6n5wpzbm6yfkngo.devices.all-e2e.test\",\"node\":\"t1\",\"policy\":{\"dead_peer_s\":45,\"ping_interval_s\":20,\"renew_before_valid_until_s\":300,\"stream_idle_s\":1800,\"stream_max_age_s\":14400},\"server_time\":1798761700,\"status\":\"active\",\"v\":2,\"valid_until\":1798776100}", "ok": false, "error": "bad_command"},
{"id": "line.bad.ok_proto_key", "dir": "n2d", "line": "OK {\"__proto__\":{},\"host\":\"psrhz6n5wpzbm6yfkngo.devices.all-e2e.test\",\"node\":\"t1\",\"policy\":{\"dead_peer_s\":45,\"ping_interval_s\":20,\"renew_before_valid_until_s\":300,\"stream_idle_s\":1800,\"stream_max_age_s\":14400},\"server_time\":1798761700,\"status\":\"active\",\"v\":2,\"valid_until\":1798776100}", "ok": false, "error": "bad_command"},
{"id": "line.bad.ok_constructor_key", "dir": "n2d", "line": "OK {\"constructor\":1,\"host\":\"psrhz6n5wpzbm6yfkngo.devices.all-e2e.test\",\"node\":\"t1\",\"policy\":{\"dead_peer_s\":45,\"ping_interval_s\":20,\"renew_before_valid_until_s\":300,\"stream_idle_s\":1800,\"stream_max_age_s\":14400},\"server_time\":1798761700,\"status\":\"active\",\"v\":2,\"valid_until\":1798776100}", "ok": false, "error": "bad_command"},
{"id": "line.bad.ok_tostring_key", "dir": "n2d", "line": "OK {\"host\":\"psrhz6n5wpzbm6yfkngo.devices.all-e2e.test\",\"node\":\"t1\",\"policy\":{\"dead_peer_s\":45,\"ping_interval_s\":20,\"renew_before_valid_until_s\":300,\"stream_idle_s\":1800,\"stream_max_age_s\":14400},\"server_time\":1798761700,\"status\":\"active\",\"toString\":\"x\",\"v\":2,\"valid_until\":1798776100}", "ok": false, "error": "bad_command"},
{"id": "line.bad.ok_policy_over_a_day", "dir": "n2d", "line": "OK {\"host\":\"psrhz6n5wpzbm6yfkngo.devices.all-e2e.test\",\"node\":\"t1\",\"policy\":{\"dead_peer_s\":45,\"ping_interval_s\":20,\"renew_before_valid_until_s\":300,\"stream_idle_s\":1800,\"stream_max_age_s\":1000000000000000},\"server_time\":1798761700,\"status\":\"active\",\"v\":2,\"valid_until\":1798776100}", "ok": false, "error": "bad_command"},
{"id": "line.bad.ok_missing_key", "dir": "n2d", "line": "OK {\"host\":\"psrhz6n5wpzbm6yfkngo.devices.all-e2e.test\",\"node\":\"t1\",\"server_time\":1798761700,\"status\":\"active\",\"v\":2,\"valid_until\":1798776100}", "ok": false, "error": "bad_command"},
{"id": "line.bad.ok_wrong_version", "dir": "n2d", "line": "OK {\"host\":\"psrhz6n5wpzbm6yfkngo.devices.all-e2e.test\",\"node\":\"t1\",\"policy\":{\"dead_peer_s\":45,\"ping_interval_s\":20,\"renew_before_valid_until_s\":300,\"stream_idle_s\":1800,\"stream_max_age_s\":14400},\"server_time\":1798761700,\"status\":\"active\",\"v\":1,\"valid_until\":1798776100}", "ok": false, "error": "bad_command"},
{"id": "line.bad.ok_underscore_host", "dir": "n2d", "line": "OK {\"host\":\"aaaaaaaaaaaaaaaaaaa_.devices.all-e2e.test\",\"node\":\"t1\",\"policy\":{\"dead_peer_s\":45,\"ping_interval_s\":20,\"renew_before_valid_until_s\":300,\"stream_idle_s\":1800,\"stream_max_age_s\":14400},\"server_time\":1798761700,\"status\":\"active\",\"v\":2,\"valid_until\":1798776100}", "ok": false, "error": "bad_command"},
{"id": "line.bad.ok_status", "dir": "n2d", "line": "OK {\"host\":\"psrhz6n5wpzbm6yfkngo.devices.all-e2e.test\",\"node\":\"t1\",\"policy\":{\"dead_peer_s\":45,\"ping_interval_s\":20,\"renew_before_valid_until_s\":300,\"stream_idle_s\":1800,\"stream_max_age_s\":14400},\"server_time\":1798761700,\"status\":\"denied\",\"v\":2,\"valid_until\":1798776100}", "ok": false, "error": "bad_command"},
{"id": "line.bad.ok_duplicate_key", "dir": "n2d", "line": "OK {\"host\":\"psrhz6n5wpzbm6yfkngo.devices.all-e2e.test\",\"node\":\"t1\",\"policy\":{\"dead_peer_s\":45,\"ping_interval_s\":20,\"renew_before_valid_until_s\":300,\"stream_idle_s\":1800,\"stream_max_age_s\":14400},\"server_time\":1798761700,\"status\":\"active\",\"v\":2,\"valid_until\":1798776100,\"v\":2}", "ok": false, "error": "bad_command"},
{"id": "line.bad.ok_float", "dir": "n2d", "line": "OK {\"host\":\"psrhz6n5wpzbm6yfkngo.devices.all-e2e.test\",\"node\":\"t1\",\"policy\":{\"dead_peer_s\":45,\"ping_interval_s\":20,\"renew_before_valid_until_s\":300,\"stream_idle_s\":1800,\"stream_max_age_s\":14400},\"server_time\":1798761700,\"status\":\"active\",\"v\":2,\"valid_until\":1798776100.5}", "ok": false, "error": "bad_command"},
{"id": "line.bad.statement_oversized", "dir": "n2d", "line": "STATEMENT aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.b.c", "ok": false, "error": "line_too_long"},
{"id": "line.bad.statement_not_jwt", "dir": "n2d", "line": "STATEMENT plain", "ok": false, "error": "bad_command"},
{"id": "line.bad.drain_negative", "dir": "n2d", "line": "DRAIN -1", "ok": false, "error": "bad_command"},
{"id": "line.bad.drain_leading_zero", "dir": "n2d", "line": "DRAIN 01", "ok": false, "error": "bad_command"},
{"id": "line.bad.drain_exponent", "dir": "n2d", "line": "DRAIN 1e3", "ok": false, "error": "bad_command"},
{"id": "line.bad.garbage_long", "dir": "d2n", "line": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", "ok": false, "error": "line_too_long"}
],
"ips": [
{"id": "ip.ok.0", "value": "0.0.0.0", "ok": true},
{"id": "ip.ok.1", "value": "1.2.3.4", "ok": true},
{"id": "ip.ok.2", "value": "203.0.113.9", "ok": true},
{"id": "ip.ok.3", "value": "255.255.255.255", "ok": true},
{"id": "ip.ok.4", "value": "10.0.0.1", "ok": true},
{"id": "ip.ok.5", "value": "::", "ok": true},
{"id": "ip.ok.6", "value": "::1", "ok": true},
{"id": "ip.ok.7", "value": "2001:db8::1", "ok": true},
{"id": "ip.ok.8", "value": "2001:db8:85a3::8a2e:370:7334", "ok": true},
{"id": "ip.ok.9", "value": "fe80::1", "ok": true},
{"id": "ip.ok.10", "value": "1:2:3:4:5:6:7:8", "ok": true},
{"id": "ip.ok.11", "value": "1:2:3:4:5:6:7::", "ok": true},
{"id": "ip.ok.12", "value": "::2:3:4:5:6:7:8", "ok": true},
{"id": "ip.ok.13", "value": "1::8", "ok": true},
{"id": "ip.ok.14", "value": "::ffff:1.2.3.4", "ok": true},
{"id": "ip.ok.15", "value": "64:ff9b::192.0.2.33", "ok": true},
{"id": "ip.ok.16", "value": "2001:0db8:0000:0000:0000:0000:0000:0001", "ok": true},
{"id": "ip.ok.17", "value": "a:b:c:d:e:f:0:1", "ok": true},
{"id": "ip.ok.18", "value": "1:2:3:4:5:6:1.2.3.4", "ok": true},
{"id": "ip.ok.19", "value": "::1.2.3.4", "ok": true},
{"id": "ip.bad.0", "value": "", "ok": false},
{"id": "ip.bad.1", "value": " ", "ok": false},
{"id": "ip.bad.2", "value": "1.2.3", "ok": false},
{"id": "ip.bad.3", "value": "1.2.3.4.5", "ok": false},
{"id": "ip.bad.4", "value": "256.1.1.1", "ok": false},
{"id": "ip.bad.5", "value": "1.2.3.256", "ok": false},
{"id": "ip.bad.6", "value": "01.2.3.4", "ok": false},
{"id": "ip.bad.7", "value": "1.02.3.4", "ok": false},
{"id": "ip.bad.8", "value": "1.2.3.04", "ok": false},
{"id": "ip.bad.9", "value": "1.2.3.", "ok": false},
{"id": "ip.bad.10", "value": ".1.2.3", "ok": false},
{"id": "ip.bad.11", "value": "1..2.3", "ok": false},
{"id": "ip.bad.12", "value": "a.b.c.d", "ok": false},
{"id": "ip.bad.13", "value": "1.2.3.4/24", "ok": false},
{"id": "ip.bad.14", "value": "1.2.3.4:80", "ok": false},
{"id": "ip.bad.15", "value": "[::1]", "ok": false},
{"id": "ip.bad.16", "value": "::1%eth0", "ok": false},
{"id": "ip.bad.17", "value": "fe80::1%1", "ok": false},
{"id": "ip.bad.18", "value": "2001:DB8::1", "ok": false},
{"id": "ip.bad.19", "value": "1:2:3:4:5:6:7:8:9", "ok": false},
{"id": "ip.bad.20", "value": "1:2:3:4:5:6:7", "ok": false},
{"id": "ip.bad.21", "value": "1::2::3", "ok": false},
{"id": "ip.bad.22", "value": ":1:2:3:4:5:6:7", "ok": false},
{"id": "ip.bad.23", "value": "1:2:3:4:5:6:7:", "ok": false},
{"id": "ip.bad.24", "value": "12345::1", "ok": false},
{"id": "ip.bad.25", "value": "g::1", "ok": false},
{"id": "ip.bad.26", "value": "1:2:3:4:5:6:7::8", "ok": false},
{"id": "ip.bad.27", "value": "::1.2.3", "ok": false},
{"id": "ip.bad.28", "value": "1.2.3.4::", "ok": false},
{"id": "ip.bad.29", "value": "::1.2.3.4:5", "ok": false},
{"id": "ip.bad.30", "value": "1:2:3:4:5:6:7:1.2.3.4", "ok": false},
{"id": "ip.bad.31", "value": "example.com", "ok": false},
{"id": "ip.bad.32", "value": "localhost", "ok": false},
{"id": "ip.bad.33", "value": "1.2.3.4 ", "ok": false},
{"id": "ip.bad.34", "value": " 1.2.3.4", "ok": false},
{"id": "ip.bad.35", "value": "1.2.3.4\n", "ok": false},
{"id": "ip.bad.36", "value": "0x7f.0.0.1", "ok": false},
{"id": "ip.bad.37", "value": "1e2.0.0.1", "ok": false},
{"id": "ip.bad.38", "value": "::ffff:01.2.3.4", "ok": false},
{"id": "ip.bad.39", "value": ":::", "ok": false},
{"id": "ip.bad.40", "value": "1:::2", "ok": false}
]
}
files/contracts/vectors/tokens.json 186.3 KB · too big to show here, read it on GitHub
files/contracts/vectors/transcripts.json 266.8 KB · too big to show here, read it on GitHub
files/scripts/e2e_pcap.py 814 lines · 41.1 KB
#!/usr/bin/env python3
"""Does a plaintext marker cross the tunnel server? Reads a tcpdump capture (classic pcap) and looks for markers in it.
This is the proof tool of the ALL end-to-end experiment (the 1.1 experiment plan section 2.9, test 2): the lead puts synthetic markers
into the bodies of requests and responses that go ChatGPT (or any HTTPS client) -> tunnel server -> device, captures the traffic on
the tunnel server (with its capture command) and this module says whether any marker is visible there. It is standard library
only, so it also runs from release_readiness later.
What it does, so that "no marker found" is not an empty statement:
* decodes Ethernet / raw IP / Linux cooked (SLL, SLL2) frames, IPv4 and IPv6, TCP;
* searches every marker in (a) the raw bytes of the file, (b) each TCP payload, (c) each TCP stream reassembled per direction, so a
marker split across two segments is still found;
* counts what WAS captured (flows, payload bytes per port, TLS records by type, the SNI names of the ClientHellos) and refuses to call
the capture clean when it holds no payload at all: a capture that saw nothing proves nothing;
* reads the hellos that cross in the clear with the ONE parser of the device (src/all_mcp/tls_policy.py, WP16): the version, cipher suite and selected group of every
ServerHello (and the alert of a refused handshake), and what each ClientHello offered, so the blindness gate records them in e2e-proof.json.
Not covered: pcapng files (tcpdump -w writes classic pcap) and IP fragments.
WP23 made it the library of the blindness gate, so it also:
* searches every marker in its ENCODED forms (`forms=True`, on in the command line): upper and lower case (DNS folds case), base64 at the three
alignments, URL-safe base64, hex, UTF-16, and in what a stream holds once it is unwrapped: the TLS record headers taken out (a marker split across two
records), gzip and zlib streams inflated, base64 runs decoded, DNS names joined across their labels;
* reads UDP too (DNS on the node's resolver port is where a node could leak a marker without a TCP connection of its own);
* refuses a capture that lost something: a hole in a flow's sequence numbers, a flow whose first or last bytes are missing, a packet cut by the snap length,
a packet count that differs from what tcpdump says it captured, or kernel drops in tcpdump's own counters, is INCOMPLETE and never CLEAN (a marker may
have been in the part that is missing);
* lists every TLS flow with its SNI, the ServerHello's version, cipher and group, and the bytes each way (`flow_table`).
python3 scripts/e2e_pcap.py FILE --marker TEXT [--marker TEXT ...] [--ports 443,8443] [--min-bytes N] [--tcpdump-log FILE] [--allow-midflow] [--exact] [--json]
Exit codes: 0 clean (markers given, none found, traffic seen, nothing missing); 5 a marker was found; 1 not conclusive (no markers given, no payload
captured, a capture that lost packets, or a damaged one); 2 unreadable file or bad usage.
"""
from __future__ import annotations
import argparse
import base64
import binascii
import dataclasses
import json
import re
import struct
import sys
import zlib
from dataclasses import dataclass, field
from pathlib import Path
from typing import Optional, Sequence
PCAP_MAGICS = {
b"\xd4\xc3\xb2\xa1": ("<", False), b"\xa1\xb2\xc3\xd4": (">", False),
b"\x4d\x3c\xb2\xa1": ("<", True), b"\xa1\xb2\x3c\x4d": (">", True),
}
PCAPNG_MAGIC = b"\x0a\x0d\x0d\x0a"
LINK_NULL, LINK_ETHERNET, LINK_RAW, LINK_SLL, LINK_SLL2 = 0, 1, 101, 113, 276
# BSD address families of a loopback frame (macOS numbers AF_INET6 as 30, FreeBSD as 28): what a `tcpdump -i lo0` file holds
BSD_FAMILY_ETHERTYPE = {2: 0x0800, 24: 0x86DD, 28: 0x86DD, 30: 0x86DD}
MAX_FILE_BYTES = 400 * 1024 * 1024
TLS_TYPES = {20: "change_cipher_spec", 21: "alert", 22: "handshake", 23: "application_data"}
MAX_EXAMPLES = 5
MAX_HELLOS_LISTED = 20
MAX_FLOWS_LISTED = 200
MAX_VIEW_BYTES = 64 * 1024 * 1024 # of one stream that is unwrapped (inflated, decoded) for the search
MAX_INFLATE_TRIES = 256 # offsets of one stream where a gzip or zlib header is tried
BASE64_RUN = re.compile(rb"[A-Za-z0-9+/_-]{24,}={0,2}")
TCP_FIN, UDP_DNS_PORT = 0x01, 53
_TLS_POLICY = None
def tls_policy():
"""The device's TLS policy module (the hello parser lives there, once): imported from this checkout's src/ the first time it is needed."""
global _TLS_POLICY
if _TLS_POLICY is None:
src = str(Path(__file__).resolve().parents[1] / "src")
if src not in sys.path:
sys.path.insert(0, src)
from all_mcp import tls_policy as module
_TLS_POLICY = module
return _TLS_POLICY
class PcapError(Exception):
"""The file cannot be read as a classic pcap capture."""
@dataclass
class Segment:
src: str
sport: int
dst: str
dport: int
seq: int
syn: bool
payload: bytes
fin: bool = False
proto: str = "tcp"
ack: int = 0
has_ack: bool = False
@dataclass
class MarkerResult:
marker: str # what is shown: the marker itself, or the label of a secret (its value is never stored or printed)
secret: bool = False
raw_hits: int = 0
packet_hits: int = 0
stream_hits: int = 0
where: list = field(default_factory=list)
forms: list = field(default_factory=list) # the forms in which it was found ("raw", "base64+1", "tls-records", ...)
flows: list = field(default_factory=list) # the connections it was found in, as flow_key() writes them
@property
def found(self) -> bool:
return bool(self.raw_hits or self.packet_hits or self.stream_hits)
@dataclass
class PcapReport:
file: str = ""
link: str = ""
packets: int = 0
tcp_packets: int = 0
skipped_frames: int = 0
truncated_tail: bool = False
payload_bytes: int = 0
flows: int = 0
ports: dict = field(default_factory=dict) # port -> {"flows": n, "payload_bytes": n}
tls_records: dict = field(default_factory=dict)
tls_flows: int = 0
non_tls_flows: list = field(default_factory=list) # [{"flow": ..., "first_bytes": ...}] a plaintext start is worth a look
sni_names: dict = field(default_factory=dict)
server_hellos: list = field(default_factory=list) # [{"flow", "version", "cipher", "group", "hybrid", "hello_retry_request"}] read from the ServerHello bytes
client_hellos: list = field(default_factory=list) # [{"flow", "server_name", "offers_hybrid", "groups", ...}]
alerts: list = field(default_factory=list) # [{"flow", "alert"}] a server that refused the handshake
tls_summary: dict = field(default_factory=dict) # counts of version, cipher and group over the ServerHellos: what e2e-proof.json records
flow_table: list = field(default_factory=list) # one entry per TCP connection: SNI, TLS version, cipher and group, bytes each way, records
udp_packets: int = 0
udp_payload_bytes: int = 0
dns_names: dict = field(default_factory=dict) # query name -> count, read from UDP and TCP port 53
markers: list = field(default_factory=list)
min_bytes: int = 1
incomplete: list = field(default_factory=list) # [{"where", "reason"}] everything that says part of the traffic is missing from the capture
stats: dict = field(default_factory=dict) # what the capture tool itself said (tcpdump's counters), when it was given
secrets_are_markers: bool = False # the secrets are all there is to search for (a marker file read by `e2e_blindness.py analyze`): they count as the markers of the verdict
@property
def found(self) -> bool:
return any(m.found for m in self.markers)
@property
def traffic_seen(self) -> bool:
return self.payload_bytes >= self.min_bytes
@property
def verdict(self) -> str:
if self.found:
return "FOUND"
if not any(self.secrets_are_markers or not m.secret for m in self.markers):
return "NO-MARKERS" # a secret that is searched for in addition does not replace the markers the caller asked for
if self.incomplete:
return "INCOMPLETE"
if not self.traffic_seen:
return "NO-TRAFFIC"
return "CLEAN"
@property
def exit_code(self) -> int:
return {"CLEAN": 0, "FOUND": 5}.get(self.verdict, 1)
def summary(self) -> str:
ports = ", ".join(f"{p}: {v['flows']} flow(s) {v['payload_bytes']} B" for p, v in sorted(self.ports.items())) or "none"
tls = ", ".join(f"{k} {v}" for k, v in sorted(self.tls_records.items())) or "none"
text = (f"{self.packets} packet(s), {self.tcp_packets} TCP, {self.flows} flow(s), {self.payload_bytes} payload bytes; "
f"by port: {ports}; TLS records: {tls}; SNI names seen: {', '.join(sorted(self.sni_names)) or 'none'}")
if self.skipped_frames:
text += f"; {self.skipped_frames} frame(s) not decoded"
if self.truncated_tail:
text += "; the last record of the file was cut"
return text
def to_dict(self) -> dict:
data = dataclasses.asdict(self)
data.update(verdict=self.verdict, found=self.found, traffic_seen=self.traffic_seen)
for entry, result in zip(data["markers"], self.markers):
entry["found"] = result.found
return data
# --------------------------------------------------------------------------- decoding
def read_frames(data: bytes) -> tuple:
"""(link type, [frame bytes], truncated_tail) of a classic pcap file."""
link, frames, truncated, _snapped = read_frames_ex(data)
return link, frames, truncated
def read_frames_ex(data: bytes) -> tuple:
"""(link type, [frame bytes], truncated_tail, snapped): `snapped` counts the frames the capture cut short (captured length below the length on the wire)."""
if data[:4] == PCAPNG_MAGIC:
raise PcapError("this is a pcapng file; capture with `tcpdump -w` (classic pcap) or convert it with editcap -F pcap")
if len(data) < 24 or data[:4] not in PCAP_MAGICS:
raise PcapError("not a pcap capture (unknown magic number)")
endian, _nano = PCAP_MAGICS[data[:4]]
link = struct.unpack_from(endian + "I", data, 20)[0]
frames, offset, truncated, snapped = [], 24, False, 0
while offset < len(data):
if offset + 16 > len(data):
truncated = True
break
incl, orig = struct.unpack_from(endian + "II", data, offset + 8)
offset += 16
if incl > 262144 * 4 or offset + incl > len(data):
truncated = True
break
snapped += 1 if orig > incl else 0
frames.append(data[offset:offset + incl])
offset += incl
return link, frames, truncated, snapped
def _ip_version(frame: bytes, link: int) -> Optional[tuple]:
"""(ip packet bytes, ethertype) or None."""
if link == LINK_ETHERNET:
if len(frame) < 14:
return None
kind, start = struct.unpack_from(">H", frame, 12)[0], 14
while kind in (0x8100, 0x88A8) and len(frame) >= start + 4: # VLAN tags
kind, start = struct.unpack_from(">H", frame, start + 2)[0], start + 4
return frame[start:], kind
if link == LINK_SLL:
return (frame[16:], struct.unpack_from(">H", frame, 14)[0]) if len(frame) >= 16 else None
if link == LINK_SLL2:
return (frame[20:], struct.unpack_from(">H", frame, 0)[0]) if len(frame) >= 20 else None
if link == LINK_RAW and frame:
return frame, {4: 0x0800, 6: 0x86DD}.get(frame[0] >> 4, 0)
if link == LINK_NULL and len(frame) >= 4:
# the family is written in the capturing host's byte order: little-endian first (every Mac this runs on), then big-endian
family = struct.unpack_from("<I", frame, 0)[0]
if family not in BSD_FAMILY_ETHERTYPE:
family = struct.unpack_from(">I", frame, 0)[0]
return frame[4:], BSD_FAMILY_ETHERTYPE.get(family, 0)
return None
def _ip_text(raw: bytes) -> str:
if len(raw) == 4:
return ".".join(str(b) for b in raw)
return ":".join(f"{raw[i]:02x}{raw[i + 1]:02x}" for i in range(0, 16, 2))
def _ip_payload(frame: bytes, link: int, protocol: int) -> Optional[tuple]:
"""(source address, destination address, the IP payload) of an IPv4 or IPv6 frame that carries `protocol` (6 TCP, 17 UDP), or None."""
found = _ip_version(frame, link)
if not found:
return None
ip, kind = found
if kind == 0x0800 and len(ip) >= 20 and ip[0] >> 4 == 4:
header = (ip[0] & 15) * 4
total = struct.unpack_from(">H", ip, 2)[0]
fragment = struct.unpack_from(">H", ip, 6)[0] & 0x1FFF
if ip[9] != protocol or header < 20 or fragment:
return None
return _ip_text(ip[12:16]), _ip_text(ip[16:20]), ip[header:total if 0 < total <= len(ip) else len(ip)]
if kind == 0x86DD and len(ip) >= 40 and ip[0] >> 4 == 6:
length, nxt, start = struct.unpack_from(">H", ip, 4)[0], ip[6], 40
while nxt in (0, 43, 60) and len(ip) >= start + 8: # hop-by-hop, routing, destination options
nxt, start = ip[start], start + (ip[start + 1] + 1) * 8
if nxt != protocol:
return None
return _ip_text(ip[8:24]), _ip_text(ip[24:40]), ip[start:40 + length if 0 < length else len(ip)]
return None
def decode_tcp(frame: bytes, link: int) -> Optional[Segment]:
found = _ip_payload(frame, link, 6)
if not found:
return None
src, dst, tcp = found
if len(tcp) < 20:
return None
sport, dport, seq = struct.unpack_from(">HHI", tcp, 0)
offset = (tcp[12] >> 4) * 4
if offset < 20 or offset > len(tcp):
return None
ack = struct.unpack_from(">I", tcp, 8)[0]
return Segment(src, sport, dst, dport, seq, bool(tcp[13] & 0x02), bytes(tcp[offset:]), fin=bool(tcp[13] & TCP_FIN), ack=ack, has_ack=bool(tcp[13] & 0x10))
def decode_udp(frame: bytes, link: int) -> Optional[Segment]:
"""A UDP datagram as a Segment (proto "udp", no sequence number): what the node's resolver sends and receives."""
found = _ip_payload(frame, link, 17)
if not found:
return None
src, dst, udp = found
if len(udp) < 8:
return None
sport, dport, length = struct.unpack_from(">HHH", udp, 0)
return Segment(src, sport, dst, dport, 0, False, bytes(udp[8:length if 8 <= length <= len(udp) else len(udp)]), proto="udp")
# --------------------------------------------------------------------------- reassembly and TLS
def _relative(seq: int, base: int) -> int:
rel = (seq - base) & 0xFFFFFFFF
return rel - (1 << 32) if rel >= 1 << 31 else rel
def reassemble_checked(segments: Sequence[Segment], peer: Sequence[Segment] = ()) -> tuple:
"""(chunks, problems) for ONE direction of ONE flow: the payload as contiguous byte strings (a gap, i.e. a lost packet, starts a new one), and what says that
part of it is missing from the capture: `hole` (a gap between two segments), `head_missing` (the handshake was captured and the first bytes are not),
`no_syn` (the capture starts in the middle of the flow), `tail_missing` (the FIN, or what the OTHER side acknowledged, says more bytes were sent than were
captured). `peer` is the opposite direction of the same flow: its acknowledgement numbers say how much of this direction arrived."""
base, saw_syn = None, False
for seg in segments:
if seg.syn:
base, saw_syn = (seg.seq + 1) & 0xFFFFFFFF, True
break
if base is None and segments:
base = segments[0].seq
placed, fin_end = [], None
for seg in segments:
if seg.fin:
fin_end = max(fin_end or 0, _relative(seg.seq, base) + len(seg.payload))
if seg.payload:
placed.append((_relative(seg.seq, base), seg.payload))
placed.sort(key=lambda item: item[0])
chunks: list = []
first, end = None, None
for offset, payload in placed:
if end is None or offset > end:
chunks.append(bytearray(payload))
end = offset + len(payload)
first = offset if first is None else first
elif offset + len(payload) > end:
chunks[-1] += payload[end - offset:]
end = offset + len(payload)
problems: list = []
if len(chunks) > 1:
problems.append("hole")
if chunks and not saw_syn:
problems.append("no_syn")
elif chunks and first is not None and first > 0:
problems.append("head_missing")
got = end or 0
if fin_end is not None and fin_end > got:
problems.append("tail_missing")
elif base is not None:
# base is the first payload byte's number: an acknowledgement of `base + n` says n bytes arrived (an ack number of 0 is a hand-built frame's: no information)
acked = max((_relative(p.ack, base) for p in peer if p.has_ack and p.ack), default=0)
if acked > got + (1 if fin_end is not None else 0):
problems.append("tail_missing")
return [bytes(c) for c in chunks], problems
def reassemble(segments: Sequence[Segment]) -> list:
"""The payload of ONE direction of ONE flow as contiguous byte strings (a gap, i.e. a lost packet, starts a new one)."""
return reassemble_checked(segments)[0]
def tls_records(stream: bytes) -> Optional[dict]:
"""Counts the TLS records at the start of a stream, or None when the stream does not start like TLS."""
counts: dict = {}
offset = 0
while offset + 5 <= len(stream):
kind, major, length = stream[offset], stream[offset + 1], struct.unpack_from(">H", stream, offset + 3)[0]
if kind not in TLS_TYPES or major != 3 or length > 18432:
break
counts[TLS_TYPES[kind]] = counts.get(TLS_TYPES[kind], 0) + 1
offset += 5 + length
return counts or None
def client_hello_sni(stream: bytes) -> Optional[str]:
"""The server name of a TLS ClientHello at the start of a stream (what a router that never decrypts can see), or None."""
try:
if len(stream) < 6 or stream[0] != 22 or stream[5] != 1:
return None
pos = 5 + 4 + 2 + 32 # record header, handshake header, version, random
pos += 1 + stream[pos] # session id
pos += 2 + struct.unpack_from(">H", stream, pos)[0] # cipher suites
pos += 1 + stream[pos] # compression methods
end = pos + 2 + struct.unpack_from(">H", stream, pos)[0]
pos += 2
while pos + 4 <= min(end, len(stream)):
kind, size = struct.unpack_from(">HH", stream, pos)
pos += 4
if kind == 0 and size >= 5:
name_len = struct.unpack_from(">H", stream, pos + 3)[0]
return stream[pos + 5:pos + 5 + name_len].decode("ascii", errors="replace")
pos += size
except (IndexError, struct.error):
return None
return None
def read_hello(stream: bytes, label: str, report: "PcapReport", hellos: list) -> dict:
"""Adds what the start of one direction of a flow says in the clear (a ClientHello, a ServerHello, an alert) to the report, and returns it for the flow's row:
{"client": ClientHello dict, "server": the first ServerHello dict, "alert": name}. A hello that is cut or not TLS is skipped: the TLS record counts above already show it."""
tls = tls_policy()
seen: dict = {}
try:
if len(stream) > 5 and stream[0] == 22 and stream[5] == 1:
client = tls.parse_client_hello(stream)
if client is not None:
seen["client"] = client.as_dict()
if len(report.client_hellos) < MAX_HELLOS_LISTED:
report.client_hellos.append({"flow": label, **client.as_dict()})
return seen
observed = tls.observe_server(stream)
except tls.TlsParseError:
return seen
for hello in observed.hellos:
hellos.append(hello)
seen.setdefault("server", hello.as_dict())
if len(report.server_hellos) < MAX_HELLOS_LISTED:
report.server_hellos.append({"flow": label, **hello.as_dict()})
if observed.alert is not None:
seen["alert"] = observed.alert.name
if len(report.alerts) < MAX_HELLOS_LISTED:
report.alerts.append({"flow": label, "alert": observed.alert.name})
return seen
def printable(data: bytes, limit: int = 8, hide: Sequence[str] = ()) -> str:
text = "".join(chr(b) if 32 <= b < 127 else "." for b in data[:limit])
for secret in hide:
if secret:
text = text.replace(secret, "<redacted>")
return text
# --------------------------------------------------------------------------- what is searched for, and where
def needle_forms(value: str) -> list:
"""[(form, bytes)] the ways `value` can show up in a capture: as it is, upper and lower case (a DNS resolver folds case), base64 and URL-safe base64 at the three
alignments it can have inside a longer text (only the characters that depend on the marker alone are kept), hex, and UTF-16 either way."""
raw = value.encode("utf-8")
forms = [("raw", raw)]
for name, other in (("lower", raw.lower()), ("upper", raw.upper())):
if other != raw:
forms.append((name, other))
for pad in range(3):
encoded = base64.b64encode(b"\0" * pad + raw).rstrip(b"=")
needle = encoded[-(-4 * pad // 3):(4 * (pad + len(raw))) // 3]
if len(needle) >= 8:
forms.append((f"base64+{pad}", needle))
forms.append((f"base64url+{pad}", needle.replace(b"+", b"-").replace(b"/", b"_")))
forms.append(("hex", raw.hex().encode("ascii")))
if raw.hex().lower() != raw.hex().upper():
forms.append(("HEX", raw.hex().upper().encode("ascii")))
text = value
forms.append(("utf16le", text.encode("utf-16-le")))
forms.append(("utf16be", text.encode("utf-16-be")))
unique, seen = [], set()
for name, needle in forms:
if needle and needle not in seen:
seen.add(needle)
unique.append((name, needle))
return unique
def tls_body_view(stream: bytes) -> Optional[bytes]:
"""The record bodies of a stream that starts with TLS records, joined: a marker that a sender cut across two records has its halves next to each other here
(the five-byte headers between them are gone). None when the stream does not start like TLS."""
out, offset, seen = bytearray(), 0, 0
while offset + 5 <= len(stream):
kind, major, length = stream[offset], stream[offset + 1], struct.unpack_from(">H", stream, offset + 3)[0]
if kind not in TLS_TYPES or major != 3 or length > 18432:
break
out += stream[offset + 5:offset + 5 + length]
offset += 5 + length
seen += 1
return bytes(out) if seen else None
COMPRESSED_HEADER = re.compile(rb"\x1f\x8b\x08|\x78[\x01\x5e\x9c\xda]")
def inflate_views(data: bytes) -> list:
"""What the gzip and zlib streams that start anywhere in `data` hold (what could be read before the stream broke counts too: a capture can cut one short)."""
views, tries = [], 0
for found in COMPRESSED_HEADER.finditer(data):
if tries >= MAX_INFLATE_TRIES:
break
tries += 1
inflater = zlib.decompressobj(31 if data[found.start()] == 0x1F else 15)
out, pos = bytearray(), found.start()
try:
while pos < len(data) and len(out) < MAX_VIEW_BYTES and not inflater.eof:
out += inflater.decompress(data[pos:pos + 2048], MAX_VIEW_BYTES - len(out))
pos += 2048
except zlib.error:
pass
if len(out) >= 8:
views.append(bytes(out))
return views
def base64_views(data: bytes) -> list:
"""What the base64 (or URL-safe base64) texts inside `data` decode to."""
views = []
for found in BASE64_RUN.finditer(data):
text = found.group(0).rstrip(b"=").replace(b"-", b"+").replace(b"_", b"/")
text = text[:len(text) - len(text) % 4 or None] if len(text) % 4 == 1 else text
try:
views.append(binascii.a2b_base64(text + b"=" * (-len(text) % 4)))
except (binascii.Error, ValueError):
continue
return views
def derived_views(data: bytes, depth: int = 2) -> list:
"""[(name, bytes)] of what `data` holds once it is unwrapped: the TLS record headers taken out, gzip and zlib inflated, base64 decoded; each of those unwrapped
once more (a base64 text of a gzip body). Bounded: MAX_VIEW_BYTES of output in all."""
out, budget = [], MAX_VIEW_BYTES
frontier = [("", data[:MAX_VIEW_BYTES])]
for _ in range(depth):
nxt = []
for prefix, blob in frontier:
candidates = []
body = tls_body_view(blob) if not prefix else None
if body:
candidates.append(("tls-records", body))
candidates += [("inflate", view) for view in inflate_views(blob)]
candidates += [("base64", view) for view in base64_views(blob)]
for name, view in candidates:
if budget <= 0:
return out
budget -= len(view)
label = f"{prefix}>{name}" if prefix else name
out.append((label, view))
nxt.append((label, view))
frontier = nxt
return out
def dns_names(payload: bytes) -> list:
"""The names a DNS message carries in its question and record sections (compression pointers followed), or [] when it is not DNS."""
names: list = []
def read(pos: int) -> tuple:
labels, jumped, end, hops = [], False, pos, 0
while True:
length = payload[pos]
if length == 0:
return ".".join(labels), (end if jumped else pos + 1)
if length & 0xC0 == 0xC0:
target = ((length & 0x3F) << 8) | payload[pos + 1]
end = end if jumped else pos + 2
jumped, pos, hops = True, target, hops + 1
if hops > 16:
raise IndexError
continue
labels.append(payload[pos + 1:pos + 1 + length].decode("ascii", errors="replace"))
pos += 1 + length
if not jumped:
end = pos
try:
questions, answers, authority, extra = struct.unpack_from(">HHHH", payload, 4)
pos = 12
for _ in range(min(questions, 16)):
name, pos = read(pos)
names.append(name)
pos += 4
for _ in range(min(answers + authority + extra, 64)):
name, pos = read(pos)
names.append(name)
size = struct.unpack_from(">H", payload, pos + 8)[0]
pos += 10 + size
except (IndexError, struct.error):
pass
return [name for name in names if name]
def parse_tcpdump_stats(text: str) -> dict:
"""tcpdump's own counters from what it prints when it ends ("12 packets captured", "12 packets received by filter", "0 packets dropped by kernel", "0 packets
dropped by interface"): {"captured", "received", "dropped_kernel", "dropped_interface"}, only the ones the text has."""
stats = {}
for key, pattern in (("captured", r"(\d+) packets? captured"), ("received", r"(\d+) packets? received by filter"),
("dropped_kernel", r"(\d+) packets? dropped by kernel"), ("dropped_interface", r"(\d+) packets? dropped by interface")):
found = re.search(pattern, text or "")
if found:
stats[key] = int(found.group(1))
return stats
def stats_problems(stats: dict, packets: int) -> list:
"""What tcpdump's counters say about a capture of `packets` packets, as [(where, reason)]; empty when they show nothing lost."""
problems = []
for key in ("dropped_kernel", "dropped_interface"):
if stats.get(key):
problems.append(("capture counters", f"tcpdump says {stats[key]} packet(s) {key.replace('dropped_', 'dropped by ')}"))
if "captured" in stats and stats["captured"] != packets:
problems.append(("capture counters", f"tcpdump says it captured {stats['captured']} packet(s) and the file holds {packets}"))
if "received" in stats and "captured" in stats and stats["received"] != stats["captured"] and not stats.get("dropped_kernel"):
problems.append(("capture counters", f"tcpdump received {stats['received']} packet(s) by the filter and captured {stats['captured']}"))
return problems
def flow_key(src: str, sport: int, dst: str, dport: int) -> str:
"""One name for a connection, whichever direction a packet goes."""
first, second = sorted([f"{src}:{sport}", f"{dst}:{dport}"])
return f"{first} <-> {second}"
# --------------------------------------------------------------------------- analysis
def analyse(data: bytes, markers: Sequence[str] = (), *, ports: Optional[Sequence[int]] = None, min_bytes: int = 1, name: str = "",
hide: Sequence[str] = (), secret_markers: Sequence[tuple] = (), forms: bool = False, stats: Optional[dict] = None,
allow_midflow: bool = False, secrets_are_markers: bool = False) -> PcapReport:
"""`secret_markers` are (label, value) pairs searched like markers (a leaked secret is a finding) but shown only by their label. `forms` also searches every
marker in its encoded forms and in what each stream holds once unwrapped (needle_forms, derived_views). `stats` is what the capture tool said about itself
(parse_tcpdump_stats): kernel drops or a packet count that differs from the file's make the capture INCOMPLETE. `allow_midflow` accepts flows whose
handshake is not in the capture (a capture that was already running); a hole, a missing tail or a cut packet is never accepted. `secrets_are_markers` lets
secrets alone be what is searched for (their values are still never shown): without it a run that was given no marker is NO-MARKERS."""
if len(data) > MAX_FILE_BYTES:
raise PcapError(f"the capture is {len(data)} bytes; above {MAX_FILE_BYTES} it is not analysed in memory")
link, frames, truncated, snapped = read_frames_ex(data)
report = PcapReport(file=name, link={LINK_NULL: "bsd-loopback", LINK_ETHERNET: "ethernet", LINK_RAW: "raw-ip", LINK_SLL: "linux-sll",
LINK_SLL2: "linux-sll2"}.get(link, f"unsupported({link})"),
packets=len(frames), truncated_tail=truncated, min_bytes=min_bytes, stats=dict(stats or {}), secrets_are_markers=secrets_are_markers)
if report.link.startswith("unsupported"):
raise PcapError(f"link type {link} is not supported (Ethernet, raw IP, BSD loopback and Linux cooked captures are)")
wanted = set(ports) if ports else None
flows: dict = {} # (src, sport, dst, dport) -> [Segment]
segments, datagrams = [], []
for frame in frames:
seg = decode_tcp(frame, link)
if seg is None:
seg = decode_udp(frame, link)
if seg is None:
report.skipped_frames += 1
continue
if wanted is not None and seg.sport not in wanted and seg.dport not in wanted:
continue
if seg.proto == "udp":
report.udp_packets += 1
report.udp_payload_bytes += len(seg.payload)
datagrams.append(seg)
continue
report.tcp_packets += 1
segments.append(seg)
flows.setdefault((seg.src, seg.sport, seg.dst, seg.dport), []).append(seg)
results = [MarkerResult(m) for m in markers] + [MarkerResult(f"<secret: {label}>", secret=True) for label, _ in secret_markers]
values = [*markers, *(value for _, value in secret_markers)]
needles: list = [] # (result, form, bytes)
for result, value in zip(results, values):
if not value:
raise PcapError("an empty marker would match everything")
for form, needle in (needle_forms(value) if forms else [("raw", value.encode("utf-8"))]):
needles.append((result, form, needle))
for result, form, needle in needles:
hits = data.count(needle)
if hits:
result.raw_hits += hits
if form not in result.forms:
result.forms.append(form)
def search(blob: bytes, view: str, where: str, key: str, kind: str) -> None:
for result, form, needle in needles:
hits = blob.count(needle)
if not hits:
continue
if kind == "packet":
result.packet_hits += hits
else:
result.stream_hits += hits
name = form if view == "raw" else f"{view}:{form}"
if name not in result.forms:
result.forms.append(name)
if key not in result.flows and len(result.flows) < MAX_FLOWS_LISTED:
result.flows.append(key)
if len(result.where) < MAX_EXAMPLES:
result.where.append(f"{where} ({hits}x{'' if view == 'raw' else ', ' + view}{'' if form == 'raw' else ', ' + form})")
connections = set()
for seg in segments:
report.payload_bytes += len(seg.payload)
if seg.payload:
search(seg.payload, "raw", f"one packet {seg.src}:{seg.sport} -> {seg.dst}:{seg.dport}", flow_key(seg.src, seg.sport, seg.dst, seg.dport), "packet")
low, high = sorted([(seg.src, seg.sport), (seg.dst, seg.dport)])
connections.add((low, high))
report.flows = len(connections)
def service_ports(a: int, b: int) -> set:
# with a --ports filter: the filtered ports of this connection; without: the lower port is the service, the other an ephemeral one
return {p for p in (a, b) if p in wanted} if wanted is not None else {min(a, b)}
for low, high in connections:
for port in service_ports(low[1], high[1]):
report.ports.setdefault(port, {"flows": 0, "payload_bytes": 0})["flows"] += 1
server_hellos: list = []
table: dict = {} # flow_key -> the row of flow_table
for (src, sport, dst, dport), segs in flows.items():
chunks, problems = reassemble_checked(segs, flows.get((dst, dport, src, sport), ()))
if (dst, dport, src, sport) not in flows and any(seg.has_ack and seg.ack for seg in segs):
problems.append("direction_missing") # these segments acknowledge bytes of the other direction, which is not in the capture at all (an ack number of 0 is a hand-built frame's)
label = f"{src}:{sport} -> {dst}:{dport}"
key = flow_key(src, sport, dst, dport)
for port in service_ports(sport, dport):
report.ports[port]["payload_bytes"] += sum(len(c) for c in chunks)
for problem in problems:
if problem == "no_syn" and allow_midflow:
continue
report.incomplete.append({"where": label, "reason": problem})
for chunk in chunks:
search(chunk, "raw", f"stream {label}", key, "stream")
if forms:
for view, blob in derived_views(chunk):
search(blob, view, f"stream {label}", key, "stream")
row = table.setdefault(key, {"flow": key, "client": None, "server": None, "sni": None, "tls": None, "alert": None, "offered": None,
"bytes": {}, "tls_records": {}, "server_port": min(sport, dport)})
row["bytes"][f"{src}:{sport}"] = sum(len(c) for c in chunks)
if chunks:
head = chunks[0]
counts = tls_records(head)
if counts:
report.tls_flows += 1
for k, v in counts.items():
report.tls_records[k] = report.tls_records.get(k, 0) + v
row["tls_records"][k] = row["tls_records"].get(k, 0) + v
sni = client_hello_sni(head)
if sni:
report.sni_names[sni] = report.sni_names.get(sni, 0) + 1
row["sni"], row["client"], row["server"] = sni, f"{src}:{sport}", f"{dst}:{dport}"
seen = read_hello(head, label, report, server_hellos)
row["offered"] = seen.get("client") or row["offered"]
row["tls"] = seen.get("server") or row["tls"]
row["alert"] = seen.get("alert") or row["alert"]
elif len(report.non_tls_flows) < 10:
report.non_tls_flows.append({"flow": label, "first_bytes": printable(head, hide=hide)})
for dgram in datagrams:
key = flow_key(dgram.src, dgram.sport, dgram.dst, dgram.dport)
where = f"datagram {dgram.src}:{dgram.sport} -> {dgram.dst}:{dgram.dport}"
search(dgram.payload, "raw", where, key, "packet")
if UDP_DNS_PORT in (dgram.sport, dgram.dport):
for dns_name in dns_names(dgram.payload):
report.dns_names[dns_name] = report.dns_names.get(dns_name, 0) + 1
if forms:
search(dns_name.replace(".", "").encode("ascii", errors="replace"), "dns-labels", where, key, "packet")
search(dns_name.encode("ascii", errors="replace"), "dns-name", where, key, "packet")
report.flow_table = sorted(table.values(), key=lambda row: row["flow"])[:MAX_FLOWS_LISTED]
report.tls_summary = tls_policy().summarize(server_hellos)
if truncated:
report.incomplete.append({"where": "capture file", "reason": "the last record of the file is cut"})
if snapped:
report.incomplete.append({"where": "capture file", "reason": f"{snapped} packet(s) were cut by the snap length"})
for where, reason in stats_problems(report.stats, len(frames)):
report.incomplete.append({"where": where, "reason": reason})
report.markers = results
return report
def analyse_file(path: Path, markers: Sequence[str] = (), **kwargs) -> PcapReport:
try:
data = Path(path).read_bytes()
except OSError as exc:
raise PcapError(f"cannot read {path}: {exc.strerror or exc}") from None
return analyse(data, markers, name=str(path), **kwargs)
def describe(report: PcapReport) -> list:
"""Human lines: the verdict first, then the evidence."""
lines = [f"CAPTURE {report.verdict}: " + {
"CLEAN": f"{len(report.markers)} marker(s)" + (" (secrets included, never shown)" if any(m.secret for m in report.markers) else "") + f" searched in {report.payload_bytes} payload bytes (raw file, every packet, every "
f"reassembled stream): none found",
"FOUND": "a marker (or a secret) crossed the capture point in the clear",
"NO-TRAFFIC": f"no TCP payload was captured ({report.payload_bytes} bytes < {report.min_bytes}): the absence of a marker proves nothing",
"NO-MARKERS": "no marker was given, so nothing was searched for",
"INCOMPLETE": f"the capture lost part of the traffic ({len(report.incomplete)} finding(s) below): a marker could have been in the part that is missing, "
f"so it proves nothing",
}[report.verdict], " " + report.summary()]
for finding in report.incomplete[:10]:
lines.append(f" incomplete: {finding['where']}: {finding['reason']}")
for result in report.markers:
state = "FOUND" if result.found else "absent"
lines.append(f" marker {result.marker!r}: {state} (raw file {result.raw_hits}, packets {result.packet_hits}, "
f"streams {result.stream_hits})" + (f" as {', '.join(result.forms)}" if result.forms else "") + ("; " + "; ".join(result.where) if result.where else ""))
for hello in report.server_hellos:
lines.append(f" ServerHello {hello['flow']}: {hello['version']} {hello['cipher']} group {hello['group']}" + (" (hello retry request)" if hello["hello_retry_request"] else ""))
for alert in report.alerts:
lines.append(f" alert {alert['flow']}: {alert['alert']}")
for row in report.flow_table:
if row["tls"]:
lines.append(f" flow {row['flow']}: SNI {row['sni'] or '-'}, {row['tls']['version']} {row['tls']['cipher']} group {row['tls']['group']}")
for flow in report.non_tls_flows:
lines.append(f" flow that does not start like TLS: {flow['flow']} first bytes {flow['first_bytes']!r}")
return lines
def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("file", type=Path)
parser.add_argument("--marker", action="append", default=[], help="plaintext marker to look for (repeatable)")
parser.add_argument("--ports", default="", help="only flows that use one of these TCP ports (default: all TCP)")
parser.add_argument("--min-bytes", type=int, default=1, help="TCP payload bytes the capture must hold to count as traffic")
parser.add_argument("--tcpdump-log", type=Path, help="what tcpdump printed when it ended (its packet counters): kernel drops make the capture INCOMPLETE")
parser.add_argument("--allow-midflow", action="store_true", help="accept flows whose handshake is not in the capture (a capture that was already running)")
parser.add_argument("--exact", action="store_true", help="search the marker as it is only, not in its encoded forms (base64, hex, case, unwrapped records, inflated streams)")
parser.add_argument("--json", action="store_true")
args = parser.parse_args(argv)
try:
ports = [int(p) for p in args.ports.split(",") if p.strip()]
stats = parse_tcpdump_stats(args.tcpdump_log.read_text(errors="replace")) if args.tcpdump_log else None
report = analyse_file(args.file, args.marker, ports=ports or None, min_bytes=args.min_bytes, forms=not args.exact, stats=stats, allow_midflow=args.allow_midflow)
except (PcapError, ValueError, OSError) as exc:
print(f"ERROR: {exc}", file=sys.stderr)
return 2
print(json.dumps(report.to_dict(), indent=2) if args.json else "\n".join(describe(report)))
return report.exit_code
if __name__ == "__main__":
sys.exit(main())
files/src/all_mcp/e2e_cert.py 1530 lines · 82.9 KB
"""ALL 1.1 end-to-end experiment (plan-all-1.1.md, 2.9.1): the device's OWN TLS certificate.
The private key is generated here, on the device, written 0600 under ``state_dir()/tls/`` and never leaves it. Only a CSR (public key and a
name) goes to the certificate authority, over ACME HTTP-01 (RFC 8555): the CA asks ``http://<name>/.well-known/acme-challenge/<token>`` on
port 80, the tunnel server forwards that plain HTTP request to this device through the tunnel, and the device answers it
(e2e_tunnel_v1.py, ChallengeStore). The CA never sees the key, and neither does the tunnel server.
Pieces
* ChallengeStore the tokens the ACME responder of the tunnel may answer (thread-safe, short-lived).
* AcmeClient a minimal RFC 8555 client (account, order, http-01, finalize, download) written with ``httpx`` and ``cryptography``, which
the agent already depends on: the experiment adds no package to uv.lock (no ``acme``, no ``josepy``).
* CertManager key + CSR + issue + renew + store. ``ensure()`` reuses a stored certificate until two thirds of its life have passed
(60 days of 90, the plan's rule: the ratio, not a day count, so shorter lifetimes keep working) and then asks for a new one.
The CA is one of: ``staging`` and ``production`` (Let's Encrypt), a directory URL (tests, Pebble), or ``selfsigned`` (a local development
mode with no network at all: the runner's ``--local`` and the tests). A certificate is stored together with the CA it came from, so a
staging certificate is never served as if it were a production one and the other way round.
ALL 2.0 adds DeviceCertManager (below): a NEW key for every issuance, kept in a generation folder (tls_store.py), DNS-01 through the control plane's
broker (reserve first, confirm after), ARI, the CA's own clock, rate-limit back-off, revokeCert at uninstall. HTTP-01 and CertManager stay for the
experiment's runner and tunnel v1 only.
Accepting the CA's terms of service is a decision of the person running the experiment: an account is created only with agree_tos=True
(the runner's --agree-tos). Nothing here runs at import time and nothing here touches the network unless issue() is called.
"""
from __future__ import annotations
import asyncio
import contextlib
import datetime
import hashlib
import json
import logging
import os
import re
import secrets
import threading
import time
from dataclasses import dataclass, replace
from pathlib import Path
from typing import Any, Callable
from urllib.parse import urlsplit
from email.utils import parsedate_to_datetime
import httpx
from cryptography import x509
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import ec, rsa
from cryptography.hazmat.primitives.asymmetric.utils import decode_dss_signature
from cryptography.x509.oid import NameOID
from . import state_io
from .config import state_dir
from ._device_storage import LockTimeout
from .tls_store import (CertInfo, Generation, TlsStore, TlsStoreError, b64url, certificate_policy, chain_matches_key, # noqa: F401 (some are re-exported)
inspect_chain, key_pem, leaf_der, parse_chain, policy as tls_policy, spki_sha256)
log = logging.getLogger("all_mcp.e2e_cert")
ACME_DIRECTORIES = {
"staging": "https://acme-staging-v02.api.letsencrypt.org/directory",
"production": "https://acme-v02.api.letsencrypt.org/directory",
}
SELF_SIGNED = "selfsigned"
USER_AGENT = "all-mcp-e2e-experiment/1.1 (+https://all-mcp.fyi)"
KEY_TYPES = ("ec", "rsa")
CHALLENGE_TTL_SECONDS = 1800.0
TOKEN_RE = re.compile(r"[A-Za-z0-9_-]{1,128}")
HOSTNAME_RE = re.compile(r"(?=.{1,253}$)([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)(\.[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)+")
MAX_ACME_BODY = 256 * 1024
class AcmeError(Exception):
"""The CA refused or the flow could not finish. ``type`` is the ACME problem type (urn:ietf:params:acme:error:...), when there was one."""
def __init__(self, message: str, *, type: str = "", status: int = 0, retry_after: float | None = None):
super().__init__(message)
self.type = type
self.status = status
self.retry_after = retry_after
# ---------------------------------------------------------------------------------------------------------------- files, keys, CSR
def tls_dir() -> Path:
return state_dir() / "tls"
def _is_windows() -> bool:
return os.name == "nt"
def _windows_owner_only(path: Path, *, directory: bool) -> None:
"""On Windows ``os.chmod`` and ``mkdir(mode=)`` restrict nothing: the folder or file would keep the ACL it inherits (Users can read it). Cut the
inheritance and grant only the current user and SYSTEM (the service may run as either): ``icacls /inheritance:r /grant:r``. A failure is raised:
a private key must not be left readable because the lock-down did not work."""
import subprocess
flags = 0x08000000 # CREATE_NO_WINDOW
who = subprocess.run(["whoami", "/user", "/fo", "csv", "/nh"], capture_output=True, text=True, timeout=15, creationflags=flags)
match = re.search(r"S-1-[0-9-]+", who.stdout or "")
if who.returncode != 0 or not match:
raise OSError("could not tell which user this is, so the key's folder was not locked to it")
rights = "(OI)(CI)F" if directory else "F"
done = subprocess.run(["icacls", str(path), "/inheritance:r", "/grant:r", f"*{match.group(0)}:{rights}", f"*S-1-5-18:{rights}"],
capture_output=True, text=True, timeout=30, creationflags=flags)
if done.returncode != 0:
raise OSError(f"icacls could not lock {path.name} to its owner: {(done.stdout or done.stderr or '').strip()[:200]}")
def _private_dir(path: Path) -> Path:
"""The folder is created 0700 (on Windows: owner-only ACL); one that already exists keeps its mode (it may not be ours to change), the files inside are private anyway."""
made = [folder for folder in (path, *path.parents) if not folder.exists()] # mkdir(parents=True) makes tls/ as well as gen-<n>/: both are ours
path.mkdir(parents=True, exist_ok=True, mode=0o700)
for folder in reversed(made):
if _is_windows():
_windows_owner_only(folder, directory=True)
elif folder == path:
os.chmod(folder, 0o700) # mkdir's mode is cut by the umask
return path
def write_private(path: Path, data: bytes) -> None:
"""Write ``data`` to ``path`` atomically and readable by this user only from the first byte: the file is CREATED 0600, so there is no moment
in which a private key sits on disk with wider permissions (a chmod after a write would leave one)."""
_private_dir(path.parent)
temporary = path.with_name(f".{path.name}.{secrets.token_hex(4)}.tmp")
descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
try:
with os.fdopen(descriptor, "wb") as stream:
stream.write(data)
stream.flush()
os.fsync(stream.fileno())
if _is_windows():
_windows_owner_only(temporary, directory=False)
state_io.replace_retrying(temporary, path)
finally:
try:
os.unlink(temporary)
except FileNotFoundError:
pass
def ensure_private_mode(path: Path) -> None:
"""A key file that is readable by group or others (copied, restored from a backup, a bad umask) is tightened to 0600 and logged."""
if _is_windows():
_windows_owner_only(path, directory=False) # no mode bits to read there: set the ACL again, it is idempotent
return
mode = path.stat().st_mode & 0o777
if mode & 0o077:
log.warning("%s was readable by others (mode %o): set to 0600", path.name, mode)
os.chmod(path, 0o600)
def generate_key(key_type: str = "ec"):
if key_type == "ec":
return ec.generate_private_key(ec.SECP256R1())
if key_type == "rsa":
return rsa.generate_private_key(public_exponent=65537, key_size=2048)
raise ValueError(f"key type must be one of {KEY_TYPES}")
def load_or_create_key(path: Path, key_type: str = "ec"):
"""The device key. Created once (0600) and kept: a restart reuses it, so the stored certificate keeps matching it."""
if path.exists():
ensure_private_mode(path)
return serialization.load_pem_private_key(path.read_bytes(), password=None)
key = generate_key(key_type)
write_private(path, key_pem(key))
return key
def make_csr(key, hostname: str) -> bytes:
"""DER CSR for one name. It carries the public key and the name, nothing else; the CA never receives the private key."""
subject = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, hostname)] if len(hostname) <= 64 else [])
builder = (x509.CertificateSigningRequestBuilder().subject_name(subject)
.add_extension(x509.SubjectAlternativeName([x509.DNSName(hostname)]), critical=False))
return builder.sign(key, hashes.SHA256()).public_bytes(serialization.Encoding.DER)
def self_signed(key, hostname: str, *, days: int = 30, now: float | None = None) -> bytes:
"""A self-signed certificate (PEM) for local development and tests. Never what ChatGPT trusts: only the runner's --local mode uses it."""
moment = datetime.datetime.fromtimestamp(now if now is not None else time.time(), datetime.timezone.utc)
name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, hostname[:64])])
certificate = (x509.CertificateBuilder().subject_name(name).issuer_name(name).public_key(key.public_key())
.serial_number(x509.random_serial_number())
.not_valid_before(moment - datetime.timedelta(minutes=5)).not_valid_after(moment + datetime.timedelta(days=days))
.add_extension(x509.SubjectAlternativeName([x509.DNSName(hostname)]), critical=False)
.add_extension(x509.BasicConstraints(ca=False, path_length=None), critical=True)
.sign(key, hashes.SHA256()))
return certificate.public_bytes(serialization.Encoding.PEM)
# ---------------------------------------------------------------------------------------------------------------- the responder's tokens
class ChallengeStore:
"""HTTP-01 answers the device's port-80 responder may give: token -> key authorization. Shared between the thread that runs ACME and the
event loop that answers the CA, so it is locked; entries die after CHALLENGE_TTL_SECONDS even if nobody removes them."""
def __init__(self, ttl: float = CHALLENGE_TTL_SECONDS, clock: Callable[[], float] = time.monotonic):
self._ttl = ttl
self._clock = clock
self._lock = threading.Lock()
self._items: dict[str, tuple[str, float]] = {}
def publish(self, token: str, key_authorization: str) -> None:
if not TOKEN_RE.fullmatch(token):
raise ValueError("invalid ACME token")
with self._lock:
self._items[token] = (key_authorization, self._clock() + self._ttl)
def unpublish(self, token: str) -> None:
with self._lock:
self._items.pop(token, None)
def get(self, token: str) -> str | None:
with self._lock:
row = self._items.get(token)
if row is None:
return None
if row[1] <= self._clock():
del self._items[token]
return None
return row[0]
def __len__(self) -> int:
with self._lock:
return len(self._items)
# ---------------------------------------------------------------------------------------------------------------- ACME (RFC 8555)
def jwk_of(public_key: ec.EllipticCurvePublicKey) -> dict[str, str]:
numbers = public_key.public_numbers()
return {"crv": "P-256", "kty": "EC", "x": b64url(numbers.x.to_bytes(32, "big")), "y": b64url(numbers.y.to_bytes(32, "big"))}
def jwk_thumbprint(jwk: dict[str, str]) -> str:
"""RFC 7638: SHA-256 of the required members in lexicographic order, no whitespace."""
canonical = json.dumps({k: jwk[k] for k in ("crv", "kty", "x", "y")}, sort_keys=True, separators=(",", ":"))
return b64url(hashlib.sha256(canonical.encode("ascii")).digest())
def key_authorization(token: str, jwk: dict[str, str]) -> str:
return f"{token}.{jwk_thumbprint(jwk)}"
def _es256(key: ec.EllipticCurvePrivateKey, signing_input: bytes) -> bytes:
r, s = decode_dss_signature(key.sign(signing_input, ec.ECDSA(hashes.SHA256())))
return r.to_bytes(32, "big") + s.to_bytes(32, "big") # JWS wants the raw pair, not DER
def _retry_after(response: httpx.Response) -> float | None:
"""Retry-After is a number of seconds or an HTTP date (then measured from the CA's own Date header, not from this computer's clock)."""
value = response.headers.get("retry-after", "").strip()
if value.isascii() and value.isdigit(): # str.isdigit() is also true for "²", which float() refuses
return float(value)
moment, now = _http_date(value), _http_date(response.headers.get("date", ""))
return max(0.0, moment - now) if moment is not None and now is not None else None
def _http_date(value: str) -> float | None:
try:
return parsedate_to_datetime(value).timestamp() if value else None
except (TypeError, ValueError):
return None
def _problem(response: httpx.Response) -> AcmeError:
kind, detail = "", ""
try:
document = response.json()
if isinstance(document, dict):
kind = str(document.get("type") or "")[:200]
detail = str(document.get("detail") or "")[:300]
except ValueError:
pass
return AcmeError(f"ACME {response.status_code} {kind.rsplit(':', 1)[-1] or 'error'}: {detail or response.reason_phrase}".strip(),
type=kind, status=response.status_code, retry_after=_retry_after(response))
class AcmeClient:
"""RFC 8555 over a synchronous httpx client. One instance, one thread at a time (CertManager calls it from a worker thread so the event loop
stays free to answer the CA through the tunnel while it validates)."""
def __init__(self, directory_url: str, account_key: ec.EllipticCurvePrivateKey, *, http: httpx.Client | None = None, email: str | None = None,
agree_tos: bool = False, account_url: str | None = None, on_account: Callable[[str], None] | None = None,
sleep: Callable[[float], None] = time.sleep, clock: Callable[[], float] = time.monotonic,
poll_interval: float = 2.0, poll_timeout: float = 180.0, on_date: Callable[[float], None] | None = None):
if urlsplit(directory_url).scheme not in {"https", "http"}:
raise ValueError("invalid ACME directory URL")
self.directory_url = directory_url
self._key = account_key
self._jwk = jwk_of(account_key.public_key())
self._http = http or httpx.Client(timeout=httpx.Timeout(30.0), headers={"User-Agent": USER_AGENT}, follow_redirects=False)
self._email = email
self._agree = agree_tos
self.account_url = account_url
self._on_account = on_account
self._sleep = sleep
self._clock = clock
self._poll_interval = poll_interval
self._poll_timeout = poll_timeout
self._on_date = on_date
self.last_date: float | None = None # the CA's own clock, from the Date header of its last answer
# True from the moment an order of this client may have a certificate (its authorizations are valid, so the CSR is sent or the order is already valid) until
# the next obtain(): an order that failed before that moment can have no certificate, and the broker is told so (HttpBroker.abandon).
self.order_may_be_issued = False
self._directory: dict | None = None
self._nonce: str | None = None
# -- transport
def _seen(self, response: httpx.Response) -> None:
moment = _http_date(response.headers.get("date", ""))
if moment is not None:
self.last_date = moment
if self._on_date:
self._on_date(moment)
def _get(self, url: str) -> httpx.Response:
response = self._http.get(url)
self._seen(response)
if response.status_code >= 400:
raise _problem(response)
return response
def directory(self) -> dict:
if self._directory is None:
document = self._get(self.directory_url).json()
for name in ("newNonce", "newAccount", "newOrder"):
if not isinstance(document, dict) or not isinstance(document.get(name), str):
raise AcmeError(f"ACME directory has no {name}")
self._directory = document
return self._directory
def _fresh_nonce(self) -> str:
response = self._http.head(self.directory()["newNonce"])
self._seen(response)
nonce = response.headers.get("replay-nonce")
if response.status_code >= 400 or not nonce:
raise AcmeError("ACME server gave no nonce", status=response.status_code)
return nonce
def _post(self, url: str, payload: dict | None, *, use_account: bool = True, accept: str | None = None) -> httpx.Response:
"""A signed POST. ``payload`` None is POST-as-GET. A badNonce answer carries a fresh nonce and is retried, as the RFC asks."""
for attempt in range(4):
nonce, self._nonce = self._nonce or self._fresh_nonce(), None
protected: dict = {"alg": "ES256", "nonce": nonce, "url": url}
if use_account:
if not self.account_url:
raise AcmeError("ACME account not created yet")
protected["kid"] = self.account_url
else:
protected["jwk"] = self._jwk
protected64 = b64url(json.dumps(protected, separators=(",", ":")).encode())
payload64 = "" if payload is None else b64url(json.dumps(payload, separators=(",", ":")).encode())
signature = _es256(self._key, f"{protected64}.{payload64}".encode("ascii"))
body = json.dumps({"protected": protected64, "payload": payload64, "signature": b64url(signature)})
headers = {"Content-Type": "application/jose+json"}
if accept:
headers["Accept"] = accept
response = self._http.post(url, content=body, headers=headers)
self._seen(response)
if len(response.content) > MAX_ACME_BODY:
raise AcmeError("ACME answer too large")
self._nonce = response.headers.get("replay-nonce") or None
if response.status_code < 400:
return response
problem = _problem(response)
if problem.type.endswith(":badNonce") and attempt < 3:
continue
raise problem
raise AcmeError("ACME badNonce repeated")
# -- account
def ensure_account(self) -> str:
if self.account_url:
return self.account_url
if not self._agree:
raise AcmeError("the CA's terms of service were not accepted (agree_tos=False): nothing was sent to the CA")
payload: dict = {"termsOfServiceAgreed": True}
if self._email:
payload["contact"] = [f"mailto:{self._email}"]
response = self._post(self.directory()["newAccount"], payload, use_account=False)
location = response.headers.get("location")
if not location:
raise AcmeError("ACME account has no Location")
self.account_url = location
if self._on_account:
self._on_account(location)
return location
# -- order
def _poll(self, url: str, done: tuple[str, ...], what: str) -> dict:
deadline = self._clock() + self._poll_timeout
while True:
document = self._post(url, None).json()
status = document.get("status")
if status in done:
return document
if status == "invalid":
detail = ""
for challenge in document.get("challenges") or []:
error = challenge.get("error") or {}
if error:
detail = f" ({str(error.get('type', '')).rsplit(':', 1)[-1]}: {str(error.get('detail', ''))[:200]})"
raise AcmeError(f"ACME {what} became invalid{detail}", type="invalid")
if self._clock() >= deadline:
raise AcmeError(f"ACME {what} did not finish in {self._poll_timeout:.0f}s (status {status})", type="timeout")
self._sleep(self._poll_interval)
def _authorize(self, url: str, publish: Callable[[str, str], None], unpublish: Callable[[str], None], challenge_type: str = "http-01") -> None:
authorization = self._post(url, None).json()
if authorization.get("status") == "valid":
return
challenge = next((c for c in authorization.get("challenges") or [] if c.get("type") == challenge_type), None)
if challenge is None:
raise AcmeError(f"the CA offered no {challenge_type} challenge")
token = str(challenge.get("token") or "")
if not TOKEN_RE.fullmatch(token):
raise AcmeError("the CA sent an invalid challenge token")
authorized = key_authorization(token, self._jwk)
# http-01 publishes the key authorization itself; dns-01 publishes its SHA-256 (RFC 8555 8.4), the 43-character TXT value
proof = authorized if challenge_type == "http-01" else b64url(hashlib.sha256(authorized.encode("ascii")).digest())
try:
publish(token, proof)
self._post(str(challenge["url"]), {}) # "I am ready": the CA now looks (through the tunnel, or at the TXT record)
self._poll(url, ("valid",), "authorization")
finally:
unpublish(token)
def obtain(self, hostname: str, csr_der: bytes, publish: Callable[[str, str], None], unpublish: Callable[[str], None], *,
challenge: str = "http-01", profile: str | None = None, replaces: str | None = None) -> bytes:
"""Order, prove control of ``hostname``, finalize with the CSR and return the PEM chain (leaf first). ``publish(token, proof)`` puts the proof where
the CA will look (http-01: the key authorization on the responder; dns-01: the TXT value) and ``unpublish(token)`` takes it away, always.
``profile`` and ``replaces`` (the ARI certificate id of the certificate this order renews) go into the order when given."""
self.ensure_account()
self.order_may_be_issued = False
order_request: dict = {"identifiers": [{"type": "dns", "value": hostname}]}
if profile:
order_request["profile"] = profile
if replaces:
order_request["replaces"] = replaces
response = self._post(self.directory()["newOrder"], order_request)
order, order_url = response.json(), response.headers.get("location")
if not order_url:
raise AcmeError("ACME order has no Location")
for authorization_url in order.get("authorizations") or []:
self._authorize(str(authorization_url), publish, unpublish, challenge)
order = self._poll(order_url, ("ready", "valid"), "order")
self.order_may_be_issued = True
if order.get("status") != "valid":
self._post(str(order["finalize"]), {"csr": b64url(csr_der)})
order = self._poll(order_url, ("valid",), "order")
certificate_url = order.get("certificate")
if not certificate_url:
raise AcmeError("ACME order is valid but has no certificate")
pem = self._post(str(certificate_url), None, accept="application/pem-certificate-chain").content
parse_chain(pem)
return pem
# -- ALL 2.0
def sync_time(self) -> float | None:
"""The CA's clock now: one HEAD on its nonce endpoint, read from the Date header. None when the CA sends no Date."""
self._nonce = self._fresh_nonce()
return self.last_date
def offered_profile(self, preferred: str, fallback: str) -> str | None:
"""``preferred`` when the directory lists it among its profiles, else ``fallback`` when listed, else None (no profile in the order)."""
profiles = (self.directory().get("meta") or {}).get("profiles")
if isinstance(profiles, dict):
for name in (preferred, fallback):
if name in profiles:
return name
return None
def renewal_info(self, certificate_id: str) -> "RenewalWindow | None":
"""ARI (RFC 9773): when the CA suggests renewing the certificate. None when the CA has no ARI or does not know the certificate."""
base = self.directory().get("renewalInfo")
if not isinstance(base, str):
return None
try:
response = self._get(base.rstrip("/") + "/" + certificate_id)
except AcmeError as exc:
if exc.status == 404:
return None
raise
document = response.json()
window = document.get("suggestedWindow") if isinstance(document, dict) else None
try:
start, end = _rfc3339(window["start"]), _rfc3339(window["end"])
except (TypeError, KeyError, ValueError):
raise AcmeError("the CA's renewal information is malformed") from None
if end <= start: # RFC 9773 4.2: an end that equals or precedes the start is invalid
raise AcmeError("the CA's renewal window ends before it starts")
return RenewalWindow(start=start, end=end, retry_after=_retry_after(response))
def revoke(self, certificate_der: bytes, reason: int = 5) -> None:
"""revokeCert signed with the ACCOUNT key (RFC 8555 7.6). Reason 5 is cessationOfOperation (uninstall). A certificate that is already revoked
is not an error."""
url = self.directory().get("revokeCert")
if not isinstance(url, str):
raise AcmeError("the CA offers no revokeCert")
try:
self._post(url, {"certificate": b64url(certificate_der), "reason": reason})
except AcmeError as exc:
if not exc.type.endswith(":alreadyRevoked"):
raise
def _rfc3339(text: str) -> float:
return datetime.datetime.fromisoformat(str(text).replace("Z", "+00:00")).timestamp()
@dataclass(frozen=True)
class RenewalWindow:
start: float
end: float
retry_after: float | None
def certificate_id(pem: bytes) -> str:
"""The ARI certificate identifier: base64url of the issuer's key identifier (the leaf's Authority Key Identifier), a dot, and base64url of the
serial number as DER integer content (RFC 9773 4.1). It is the value of ``replaces`` and the tail of the renewalInfo URL."""
leaf = parse_chain(pem)[0]
try:
key_id = leaf.extensions.get_extension_for_class(x509.AuthorityKeyIdentifier).value.key_identifier
except x509.ExtensionNotFound:
key_id = None
if not key_id:
raise ValueError("the certificate has no Authority Key Identifier")
serial = leaf.serial_number
return f"{b64url(key_id)}.{b64url(serial.to_bytes((serial.bit_length() + 8) // 8, 'big'))}"
# ---------------------------------------------------------------------------------------------------------------- the device's certificate
@dataclass(frozen=True)
class CertState:
chain_path: Path
key_path: Path
info: CertInfo
ca: str
issued_now: bool
generation: int = 0 # the tls/gen-<n>/ it lives in (DeviceCertManager); 0 for the 1.1 flat layout
@property
def renew_at(self) -> float:
return self.info.renew_at
class CertManager:
"""LEGACY (the 1.1 experiment: one key kept for good, flat files in tls/, HTTP-01 through the tunnel; the tunnel v1 tests still build on it). The 2.0
device uses DeviceCertManager below, which keeps the same call names (ensure, current, due, needs_terms, issue) over generations.
Key, CSR, issue, renew and store the device's certificate. Everything lives in ``directory`` (default state_dir()/tls/):
device.key (0600), fullchain.pem, certificate.json (which CA, which name), acme-account-<ca>.key and .json."""
def __init__(self, hostname: str, *, ca: str = "staging", directory: Path | None = None, key_type: str = "ec", email: str | None = None,
agree_tos: bool = False, challenges: ChallengeStore | None = None, http: httpx.Client | None = None,
clock: Callable[[], float] = time.time, sleep: Callable[[float], None] = time.sleep,
poll_interval: float = 2.0, poll_timeout: float = 180.0):
hostname = hostname.strip().lower().rstrip(".")
if not HOSTNAME_RE.fullmatch(hostname):
raise ValueError("invalid host name")
if key_type not in KEY_TYPES:
raise ValueError(f"key type must be one of {KEY_TYPES}")
self.hostname = hostname
self.ca = ca
self.directory_url = ACME_DIRECTORIES.get(ca, ca if ca != SELF_SIGNED else "")
if ca != SELF_SIGNED and not self.directory_url.startswith(("https://", "http://")):
raise ValueError("ca must be staging, production, selfsigned or an ACME directory URL")
self.dir = directory or tls_dir()
self.key_type = key_type
self.email = email
self.agree_tos = agree_tos
self.challenges = challenges if challenges is not None else ChallengeStore() # an EMPTY store is falsy (__len__): never `or`
self._http = http
self._clock = clock
self._sleep = sleep
self._poll_interval = poll_interval
self._poll_timeout = poll_timeout
self.key_path = self.dir / "device.key"
self.chain_path = self.dir / "fullchain.pem"
self.meta_path = self.dir / "certificate.json"
self._ca_tag = ca if ca in ACME_DIRECTORIES or ca == SELF_SIGNED else "custom-" + hashlib.sha256(ca.encode()).hexdigest()[:8]
self._lock = threading.Lock()
# -- what is stored
def _stored_ca(self) -> str | None:
try:
document = json.loads(self.meta_path.read_text(encoding="utf-8"))
return str(document.get("ca")) if isinstance(document, dict) else None
except (OSError, ValueError):
return None
def current(self) -> CertState | None:
"""The stored certificate when it is usable now: it covers the name, matches the key, comes from the configured CA and has not expired.
Anything else is None (and a new one is issued)."""
try:
pem = self.chain_path.read_bytes()
key = serialization.load_pem_private_key(self.key_path.read_bytes(), password=None)
info = inspect_chain(pem)
if (self.hostname not in info.names or not chain_matches_key(pem, key) or self._stored_ca() != self._ca_tag
or not info.not_before - 600 <= self._clock() < info.not_after):
return None
ensure_private_mode(self.key_path)
return CertState(self.chain_path, self.key_path, info, self._ca_tag, False)
except (OSError, ValueError):
return None
def due(self, state: CertState | None = None) -> bool:
state = state or self.current()
return state is None or self._clock() >= state.renew_at
def needs_terms(self) -> bool:
"""True when the next certificate needs a NEW account at the CA, i.e. someone has to accept the CA's terms first (never for selfsigned or when a
stored certificate is still good, and never when an account of this CA already exists here)."""
if self.ca == SELF_SIGNED:
return False
state = self.current()
if state is not None and not self.due(state):
return False
return not (self.dir / f"acme-account-{self._ca_tag}.json").exists()
# -- issuing
def _account(self) -> AcmeClient:
key_path = self.dir / f"acme-account-{self._ca_tag}.key"
url_path = self.dir / f"acme-account-{self._ca_tag}.json"
_private_dir(self.dir)
account_key = load_or_create_key(key_path, "ec")
url = None
try:
document = json.loads(url_path.read_text(encoding="utf-8"))
url = str(document["url"]) if isinstance(document, dict) and document.get("url") else None
except (OSError, ValueError, KeyError):
pass
def remember(location: str) -> None:
write_private(url_path, json.dumps({"url": location, "directory": self.directory_url}).encode())
return AcmeClient(self.directory_url, account_key, http=self._http, email=self.email, agree_tos=self.agree_tos, account_url=url,
on_account=remember, sleep=self._sleep, poll_interval=self._poll_interval, poll_timeout=self._poll_timeout)
def issue(self) -> CertState:
"""Always asks for a new certificate (ensure() decides when). The device key is created on first use and kept."""
with self._lock:
_private_dir(self.dir)
key = load_or_create_key(self.key_path, self.key_type)
if self.ca == SELF_SIGNED:
pem = self_signed(key, self.hostname, now=self._clock())
else:
log.info("asking %s for a certificate for %s", urlsplit(self.directory_url).netloc, self.hostname)
csr = make_csr(key, self.hostname)
try:
pem = self._account().obtain(self.hostname, csr, self.challenges.publish, self.challenges.unpublish)
except AcmeError as exc:
if not exc.type.endswith(":accountDoesNotExist"):
raise
# the CA no longer knows our account (Let's Encrypt staging is reset now and then): forget it and register again
log.warning("the CA does not know the stored account any more: registering a new one")
(self.dir / f"acme-account-{self._ca_tag}.json").unlink(missing_ok=True)
pem = self._account().obtain(self.hostname, csr, self.challenges.publish, self.challenges.unpublish)
info = inspect_chain(pem)
if self.hostname not in info.names or not chain_matches_key(pem, key):
raise AcmeError("the certificate does not match this device's key and name")
write_private(self.chain_path, pem)
write_private(self.meta_path, json.dumps({"ca": self._ca_tag, "hostname": self.hostname, "issued_at": int(self._clock()),
"not_after": int(info.not_after)}).encode())
return CertState(self.chain_path, self.key_path, info, self._ca_tag, True)
def ensure(self) -> CertState:
"""The certificate to serve with: the stored one while it is not due for renewal, otherwise a new one. A stored certificate that
is still valid but due is replaced only if the new one is obtained: a failed renewal raises and the old one keeps working until it expires."""
state = self.current()
if state is not None and not self.due(state):
return state
try:
return self.issue()
except Exception:
if state is not None:
expires = datetime.datetime.fromtimestamp(state.info.not_after, datetime.timezone.utc).isoformat()
log.warning("renewal failed; the current certificate stays in use until %s", expires)
raise
# ================================================================================================================ ALL 2.0
# The device's certificate manager, version 2: DNS-01 through the control plane's broker, a new key for every issuance in a generation folder
# (tls_store.py), ARI, the CA's clock for every decision, a persisted attempt counter with back-off, rate limits that keep the current certificate.
# Plan 2.0 sections 2 and 4. Every number comes from contracts/policy.json (tls_store.policy()).
BACKOFF_CEILING_S = 86400.0 # consecutive failures wait 6 h, 12 h, 24 h and then 24 h (the plan names the 6 h floor and the doubling)
SOFT_RETRY_FLOOR_S = 30.0 # a failure that cost nothing (no connection, a 5xx before any reservation) is retried soon: 30 s, doubling to 30 min
SOFT_RETRY_CEILING_S = 1800.0
BLOCK_CEILING_S = 7 * 86400.0 # no answer of a server can keep this device quiet for longer than this
CLOCK_SYNC_EVERY_S = 600.0
DIRECTORY_TTL_S = 86400.0
DAY_S = 86400.0
STATE_VERSION = 1
BROKER_BODY_MAX = 65536
WORD_RE = re.compile(r"[a-z_]{1,48}")
RESERVATION_RE = re.compile(r"[0-9a-f]{32}")
SPKI_RE = re.compile(r"[A-Za-z0-9_-]{43}")
ACME_PATH = "/v1/devices/acme/"
# ---------------------------------------------------------------------------------------------------------------- the broker
class BrokerError(Exception):
"""The broker (or the way to it) refused. ``word`` is the closed word of its error answer ("label_retired", "dns_rate_limited", ...) or one of
ours: broker_unreachable, credentials_unavailable, bad_answer. ``status`` is the HTTP status (0: no answer at all)."""
def __init__(self, word: str, *, status: int = 0, retry_after: float | None = None):
super().__init__(word)
self.word = word
self.status = status
self.retry_after = retry_after
@property
def soft(self) -> bool:
"""A failure that says nothing against this device: nobody answered, the control plane is busy or down, or it asks to wait."""
return self.status in (0, 429) or self.status >= 500
@dataclass(frozen=True)
class Reservation:
id: str
klass: str
expires_at: float
server_time: float
@dataclass(frozen=True)
class Deferral:
"""Over budget is never an error: come back at ``enroll_after`` (the broker's jitter is already in it)."""
enroll_after: float
server_time: float
@dataclass(frozen=True)
class DnsAnswer:
name: str
type: str
ttl: int
class HttpBroker:
"""The device's side of POST /v1/devices/acme/{reserve,dns,confirm,release}. Every call carries the device's own
control session and proof (``headers(url, method)``, device_credentials.control_headers by default). The answers are checked against the
contract's shapes; an answer that does not fit is ``bad_answer``, never trusted."""
def __init__(self, base_url: str, *, headers: Callable[[str, str], dict[str, str]] | None = None, http: httpx.Client | None = None,
timeout: float = 20.0):
if urlsplit(base_url).scheme not in {"https", "http"}:
raise ValueError("invalid control plane URL")
self.base = base_url.rstrip("/")
self._headers = headers or _control_headers
self._http = http or httpx.Client(timeout=httpx.Timeout(timeout), headers={"User-Agent": USER_AGENT}, follow_redirects=False)
def _call(self, method: str, operation: str, body: dict | None = None) -> dict:
url = f"{self.base}{ACME_PATH}{operation}"
try:
headers = {"Accept": "application/json", **self._headers(url, method)}
except Exception: # the credentials are not there (yet): nothing was sent, and nothing about it is worth more than a closed word
raise BrokerError("credentials_unavailable") from None
content = None
if body is not None:
headers["Content-Type"] = "application/json"
content = json.dumps(body, separators=(",", ":")).encode("utf-8")
try:
response = self._http.request(method, url, content=content, headers=headers)
except httpx.HTTPError:
raise BrokerError("broker_unreachable") from None
if len(response.content) > BROKER_BODY_MAX:
raise BrokerError("bad_answer", status=response.status_code)
try:
document = response.json()
except ValueError:
document = None
if response.status_code >= 400:
word = document.get("error") if isinstance(document, dict) else None
wait = document.get("retry_after_s") if isinstance(document, dict) else None
raise BrokerError(word if isinstance(word, str) and WORD_RE.fullmatch(word) else "broker_error", status=response.status_code,
retry_after=float(wait) if type(wait) is int else _retry_after(response))
if not isinstance(document, dict) or document.get("ok") is not True:
raise BrokerError("bad_answer", status=response.status_code)
return document
@staticmethod
def _seconds(document: dict, key: str) -> float:
value = document.get(key)
if type(value) is not int or value < 0:
raise BrokerError("bad_answer")
return float(value)
def reserve(self, spki_sha256: str, purpose: str) -> Reservation | Deferral:
document = self._call("POST", "reserve", {"v": 1, "spki_sha256": spki_sha256, "purpose": purpose})
stamp = self._seconds(document, "server_time")
if document.get("status") == "deferred":
return Deferral(self._seconds(document, "enroll_after"), stamp)
reservation_id, klass = document.get("reservation_id"), document.get("class")
if (document.get("status") != "granted" or not isinstance(reservation_id, str) or not RESERVATION_RE.fullmatch(reservation_id)
or klass not in tls_policy()["enums"]["budget_class"]):
raise BrokerError("bad_answer")
return Reservation(reservation_id, klass, self._seconds(document, "expires_at"), stamp)
def _dns(self, reservation_id: str, body: dict) -> DnsAnswer:
document = self._call("POST", "dns", {"v": 1, "reservation_id": reservation_id, **body})
name, ttl = document.get("name"), document.get("ttl")
if not isinstance(name, str) or document.get("type") != "TXT" or type(ttl) is not int or not 1 <= ttl <= 3600:
raise BrokerError("bad_answer")
return DnsAnswer(name, "TXT", ttl)
def set_txt(self, reservation_id: str, value: str) -> DnsAnswer:
return self._dns(reservation_id, {"action": "set", "value": value})
def clear_txt(self, reservation_id: str) -> DnsAnswer:
return self._dns(reservation_id, {"action": "clear"})
def confirm(self, reservation_id: str, spki_sha256: str, not_before: int, not_after: int) -> float:
document = self._call("POST", "confirm", {"v": 1, "reservation_id": reservation_id, "spki_sha256": spki_sha256, "not_before": int(not_before),
"not_after": int(not_after)})
return self._seconds(document, "server_time")
def abandon(self, reservation_id: str) -> None:
"""The order for this reservation ended before the CSR was sent: no certificate can exist for its key, and the broker frees the slot."""
self._call("POST", "abandon", {"v": 1, "reservation_id": reservation_id})
def release(self, reason: str) -> None:
self._call("POST", "release", {"v": 1, "reason": reason})
def status(self) -> dict:
return self._call("GET", "status")
def _control_headers(url: str, method: str) -> dict[str, str]:
from .device_credentials import control_headers
return control_headers(url, method)
def control_plane_url(values: dict[str, str] | None = None) -> str:
from .config import runtime_values
values = runtime_values() if values is None else values
return (values.get("ALL_CONTROL_PLANE_URL") or values.get("ALL_AI_PROXY_URL") or "https://api.all-mcp.fyi").rstrip("/")
# ---------------------------------------------------------------------------------------------------------------- the CA's clock
class CaClock:
"""Time as the CA tells it. The device's own clock can be a year off (a dead battery, a wrong time zone, a VM resumed from an old snapshot); a
certificate that "expired" on that clock must not be renewed over and over. ``observe`` takes the CA's Date header (or the broker's server_time);
from then on ``now`` is that moment plus the time elapsed on the MONOTONIC clock, so a later jump of the wall clock changes nothing."""
def __init__(self, wall: Callable[[], float] = time.time, monotonic: Callable[[], float] = time.monotonic):
self._wall = wall
self._monotonic = monotonic
self._remote: float | None = None
self._at = 0.0
def observe(self, remote: float) -> None:
self._remote, self._at = float(remote), self._monotonic()
def observe_if_stale(self, remote: float, maximum_age: float) -> None:
"""A second source (the broker's server_time, whole seconds) fills in only when the CA's own Date is not recent."""
if self.age() >= maximum_age:
self.observe(remote)
@property
def synced(self) -> bool:
return self._remote is not None
def age(self) -> float:
"""Seconds since the last observation (infinite before the first)."""
return self._monotonic() - self._at if self._remote is not None else float("inf")
def now(self) -> float:
return self._remote + (self._monotonic() - self._at) if self._remote is not None else self._wall()
@property
def offset(self) -> float | None:
"""The CA's time minus this computer's clock, in seconds; None while the CA has not been heard."""
return self.now() - self._wall() if self._remote is not None else None
# ---------------------------------------------------------------------------------------------------------------- the manager
@dataclass(frozen=True)
class PassResult:
"""What one pass of the manager did. ``action``: valid (nothing to do), issued, adopted (a certificate a crash left unnamed was put in service),
waiting (due, but not allowed yet: ``reason`` and ``wait_until``), failed (``error`` is the closed word), terms (the CA's terms are not accepted
yet, nothing was sent), offline (the CA could not be reached to learn the time), busy (another pass runs), unsafe (the storage is not trusted)."""
action: str
generation: int | None = None
wait_until: float | None = None
reason: str = ""
error: str = ""
@property
def issued(self) -> bool:
return self.action in ("issued", "adopted")
class CertUnavailable(AcmeError):
"""ensure() has no certificate to give: none was ever issued and the pass could not issue one now (``retry_after`` says how long to wait)."""
def _snake(word: str) -> str:
return re.sub(r"(?<!^)(?=[A-Z])", "_", word).lower()
def error_word(exc: BaseException) -> str:
"""The closed word that stands for a failure in the status file and the fleet heartbeat: never text from the network."""
if isinstance(exc, BrokerError):
return exc.word
if isinstance(exc, AcmeError):
tail = _snake(exc.type.rsplit(":", 1)[-1]) if exc.type else "acme_error"
return {"invalid": "validation_failed", "timeout": "acme_timeout"}.get(tail, tail if WORD_RE.fullmatch(tail) else "acme_error")
if isinstance(exc, TlsStoreError):
return exc.word
if isinstance(exc, httpx.HTTPError):
return "ca_unreachable"
if isinstance(exc, OSError):
return "storage_error"
return "internal_error"
class DeviceCertManager:
"""The device's TLS certificate, from first issuance to uninstall.
``run_once()`` is one decision pass, safe to call at any time, from any process (a second one at the same moment answers ``busy``). It never
raises for an expected condition and never touches the serving certificate unless a new one is complete and validated. The rules (plan section 4):
* the CA decides what time it is (its Date header); the device's clock only breaks ties when the CA cannot be reached;
* a certificate is due when ARI says so (checked twice a day), or when a third of its life is left (the backstop, with or without ARI), or when it
has expired;
* an attempt is counted (and written to disk) when a reservation is granted; at least min_attempt_gap_s between attempts, back-off doubling after
failures, at most max_issuances_per_day successes in 24 hours; a rate limit or an ``enroll_after`` keeps the current certificate and waits;
* every issuance: new EC P-256 key -> its SPKI hash reserved at the broker BEFORE the order -> DNS-01 through the broker -> chain written beside
the key in tls/gen-<n>/ -> validated as a pair -> pointer swapped atomically -> issuance confirmed at the broker.
Modes: ``dns-01`` (a broker is given: production), ``http-01`` (``dev_http01=True`` and no broker: the experiment's runner through its tunnel) and
``selfsigned`` (ca="selfsigned": local development and tests, no network).
"""
def __init__(self, hostname: str, *, ca: str = "staging", directory: Path | None = None, broker=None, email: str | None = None,
agree_tos: bool = False, dev_http01: bool = False, challenges: ChallengeStore | None = None, http: httpx.Client | None = None,
clock: Callable[[], float] = time.time, monotonic: Callable[[], float] = time.monotonic,
sleep: Callable[[float], None] = time.sleep, poll_interval: float = 2.0, poll_timeout: float = 180.0, dns_settle: float = 5.0,
key_type: str = "ec", faults: Callable[[str], None] | None = None, rng: Callable[[], float] = secrets.SystemRandom().random):
hostname = hostname.strip().lower().rstrip(".")
if not HOSTNAME_RE.fullmatch(hostname):
raise ValueError("invalid host name")
if key_type != "ec":
raise ValueError("device keys are EC P-256")
self.hostname = hostname
self.ca = ca
self.directory_url = ACME_DIRECTORIES.get(ca, ca if ca != SELF_SIGNED else "")
if ca != SELF_SIGNED and not self.directory_url.startswith(("https://", "http://")):
raise ValueError("ca must be staging, production, selfsigned or an ACME directory URL")
if ca == SELF_SIGNED:
self.mode = "selfsigned"
elif broker is not None:
if dev_http01:
raise ValueError("a broker and HTTP-01 together: choose one (HTTP-01 is for development only)")
if not re.fullmatch(tls_policy()["identity"]["fqdn_regex"], hostname):
raise ValueError("a device host is one label under the devices zone")
self.mode = "dns-01"
elif dev_http01:
self.mode = "http-01"
else:
raise ValueError("a broker is required: DNS-01 through the control plane is the way a device gets its certificate "
"(HTTP-01 exists only for development: dev_http01=True)")
self.broker = broker
self.email = email
self.agree_tos = agree_tos
self.challenges = challenges if challenges is not None else ChallengeStore() # an EMPTY store is falsy (__len__): never `or`
self.store = TlsStore(directory, faults=faults)
self.dir = self.store.dir
self.clock = CaClock(clock, monotonic)
self._wall = clock
self._http = http
self._sleep = sleep
self._poll_interval = poll_interval
self._poll_timeout = poll_timeout
self._dns_settle = dns_settle
self._rng = rng
self._policy = certificate_policy()
self._ca_tag = ca if ca in ACME_DIRECTORIES or ca == SELF_SIGNED else "custom-" + hashlib.sha256(ca.encode()).hexdigest()[:8]
self._client_cache: AcmeClient | None = None
self._mutex = threading.Lock() # two passes of the SAME process; across processes it is the file lock
# -- paths a consumer wants (the active generation's; before any issuance they point where the first one will not be)
@property
def chain_path(self) -> Path:
generation = self._active()
return generation.chain_path if generation else self.dir / "fullchain.pem"
@property
def key_path(self) -> Path:
generation = self._active()
return generation.key_path if generation else self.dir / "device.key"
def _active(self) -> Generation | None:
try:
return self.store.active()
except TlsStoreError:
return None
# -- the CA
def _account_files(self) -> tuple[str, str]:
return f"acme-account-{self._ca_tag}.key", f"acme-account-{self._ca_tag}.json"
def _account_url(self) -> str | None:
raw = self.store.read_file(self._account_files()[1])
try:
document = json.loads(raw.decode("utf-8")) if raw else None
except (ValueError, UnicodeError):
return None
return str(document["url"]) if isinstance(document, dict) and document.get("url") else None
def _acme(self) -> AcmeClient:
"""The CA client for this pass. The account key is created on first use (0600, in tls/) and only ever used to register and to revoke."""
if self._client_cache is not None:
return self._client_cache
key_file, url_file = self._account_files()
raw = self.store.read_file(key_file)
if raw:
account_key = serialization.load_pem_private_key(raw, password=None)
else:
account_key = generate_key("ec")
self.store.write_file(key_file, key_pem(account_key))
def remember(location: str) -> None:
self.store.write_file(url_file, json.dumps({"url": location, "directory": self.directory_url}).encode())
self._client_cache = AcmeClient(self.directory_url, account_key, http=self._http, email=self.email, agree_tos=self.agree_tos,
account_url=self._account_url(), on_account=remember, sleep=self._sleep, poll_interval=self._poll_interval,
poll_timeout=self._poll_timeout, on_date=self.clock.observe)
return self._client_cache
def needs_terms(self) -> bool:
"""True when the next certificate needs a NEW account at the CA, i.e. someone has to accept the CA's terms first."""
if self.mode == "selfsigned" or self._account_url() is not None:
return False
return self._due_reason(self._active(), self.clock.now()) is not None
# -- the pass
def run_once(self, *, force: bool = False) -> PassResult:
"""One decision pass (see the class). ``force`` skips only the question "is it due"; every wait still applies."""
if not self._mutex.acquire(blocking=False):
return PassResult("busy")
try:
with self.store.exclusive(0.0):
return self._pass(force)
except LockTimeout:
return PassResult("busy")
except TlsStoreError as exc:
if exc.word == "storage_error": # the disk, not the folder: full, read-only, an I/O error
log.warning("the TLS folder could not be written (%s)", exc.word)
return PassResult("failed", error=exc.word)
log.warning("the TLS folder is not trusted (%s): nothing was changed", exc.word)
with contextlib.suppress(Exception):
self.store.write_status(self._status_document(None, {}, self.clock.now(), "unsafe", error=exc.word))
return PassResult("unsafe", error=exc.word)
finally:
self._mutex.release()
def _pass(self, force: bool) -> PassResult:
self._client_cache = None
self.store.key_storage() # writes the descriptor on first use; refuses a descriptor that says something this version cannot honor
state = self._load_state()
if self.mode != "selfsigned" and self._account_url() is None and not self.agree_tos:
# A new account needs the person's agreement to the CA's terms (the installer's flag): until then nothing at all is sent to the CA
active = self._active()
self._save(state, active, self.clock.now(), "waiting" if active is None else "valid", reason="terms_required")
return PassResult("terms", generation=active.number if active else None, reason="terms_required")
trusted = self._sync_clock()
now = self.clock.now()
adopted = self._adopt(now, state)
for name in self.store.gc():
log.info("removed %s", name)
active = self.store.active() # storage the checks refuse raises here: nothing is repaired or replaced behind a planted link or a foreign file
self._flush_confirm(state, active)
if self.mode != "selfsigned" and active is not None and not trusted:
self._save(state, active, now, "valid", reason="ca_unreachable")
return PassResult("offline", generation=active.number, reason="ca_unreachable")
if active is not None and self.mode != "selfsigned":
self._refresh_ari(state, active, now)
reason = "forced" if force else self._due_reason(active, now, state)
if reason is None:
self._save(state, active, now, "valid")
return PassResult("adopted" if adopted else "valid", generation=active.number if active else None)
wait = self._wait(state, now) if self.mode != "selfsigned" else None
if wait is not None:
self._save(state, active, now, "waiting", reason=wait[1], wait_until=wait[0])
return PassResult("waiting", generation=active.number if active else None, wait_until=wait[0], reason=wait[1])
return self._issue(state, active, reason, now)
# -- the clock
def _sync_clock(self) -> bool:
"""Learn the CA's time (one HEAD request) unless it was learned a moment ago. False when it could not be learned and was never learned."""
if self.mode == "selfsigned" or self.clock.age() < CLOCK_SYNC_EVERY_S:
return True
try:
self._acme().sync_time()
except (AcmeError, httpx.HTTPError, ValueError) as exc:
log.info("the CA's clock could not be read (%s)", error_word(exc))
return self.clock.synced and self.clock.age() < CLOCK_SYNC_EVERY_S * 6
# -- state
def _load_state(self) -> dict[str, Any]:
try:
state = self.store.read_state()
except TlsStoreError as exc:
if exc.word != "file_invalid":
raise
state = None
corrupted = True
else:
corrupted = False
state = dict(state) if isinstance(state, dict) and state.get("v") == STATE_VERSION else {"v": STATE_VERSION}
if corrupted: # a record that cannot be read must not reset the counters to "never tried": the gap starts now
state["last_attempt"] = self.clock.now()
for key, default in (("attempt_count", 0), ("failures", 0), ("soft_failures", 0)):
state[key] = state[key] if type(state.get(key)) is int and state[key] >= 0 else default
return state
def _persist(self, state: dict[str, Any]) -> None:
self.store.write_state(state)
# -- deciding
def _due_reason(self, active: Generation | None, now: float, state: dict[str, Any] | None = None) -> str | None:
if active is None:
return "none"
if active.ca != self._ca_tag or self.hostname not in active.info.names:
return "other_ca"
if now >= active.info.not_after:
return "expired"
if now >= active.info.renew_at:
return "backstop"
ari = (state or {}).get("ari") or {}
if ari.get("fp") == active.info.fingerprint and "start" in ari and now >= ari["start"] + ari["frac"] * (ari["end"] - ari["start"]):
return "ari"
return None
def _refresh_ari(self, state: dict[str, Any], active: Generation, now: float) -> None:
"""Ask the CA when it suggests renewing, twice a day. A CA without ARI, a failure or a malformed answer changes nothing: the backstop stays."""
ari = state.get("ari") if isinstance(state.get("ari"), dict) else {}
same = ari.get("fp") == active.info.fingerprint
if same and now < ari.get("next_check", 0):
return
if now >= active.info.not_after:
return # RFC 9773 4.3: never ask about a certificate that has expired
interval = DAY_S / self._policy["ari_checks_per_day"]
record: dict[str, Any] = {"fp": active.info.fingerprint, "next_check": now + interval, "frac": ari["frac"] if same and "frac" in ari else self._rng()}
try:
window = self._acme().renewal_info(certificate_id(active.chain_pem))
except (AcmeError, httpx.HTTPError, ValueError) as exc:
log.info("ARI: %s", error_word(exc))
record.update({key: ari[key] for key in ("checked_at", "start", "end") if same and key in ari}) # keep the last window it gave
else:
if window is not None:
record.update(checked_at=now, start=window.start, end=window.end, next_check=now + max(interval, min(window.retry_after or 0.0, DAY_S))) # the policy's twice a day is the floor; a day the ceiling (RFC 9773 4.3.2)
state["ari"] = record
self._persist(state)
def _backoff(self, failures: int) -> float:
gap = float(self._policy["min_attempt_gap_s"])
return min(BACKOFF_CEILING_S, gap * 2 ** max(0, failures - 1)) if failures else gap
def _wait(self, state: dict[str, Any], now: float) -> tuple[float, str] | None:
"""The earliest moment an issuance is allowed again and why, or None when one is allowed now."""
waits: list[tuple[float, str]] = []
blocked = state.get("blocked_until")
if type(blocked) in (int, float) and blocked > now:
waits.append((min(float(blocked), now + BLOCK_CEILING_S), str(state.get("blocked_reason") or "blocked")))
soft = state.get("soft_until")
if type(soft) in (int, float) and soft > now:
waits.append((min(float(soft), now + BLOCK_CEILING_S), "retry_soon"))
last = state.get("last_attempt")
if type(last) in (int, float):
if last > now: # the clock went back: the gap starts again from now, so a wrong clock can delay but never loop
last = state["last_attempt"] = now
gap = self._backoff(state.get("failures", 0))
if last + gap > now:
waits.append((last + gap, "backoff" if state.get("failures") else "gap"))
issued = [min(float(t), now) for t in state.get("issuances", []) if type(t) in (int, float) and t > now - DAY_S]
if len(issued) >= self._policy["max_issuances_per_day"]:
waits.append((min(issued) + DAY_S, "daily_cap"))
return max(waits) if waits else None
# -- issuing
def _issue(self, state: dict[str, Any], active: Generation | None, reason: str, now: float) -> PassResult:
self._save(state, active, now, "issuing", reason=reason)
number: int | None = None
attempted = False
reservation: Reservation | None = None
client: AcmeClient | None = None
try:
if self.mode == "selfsigned":
number, key = self.store.new_generation()
chain = self_signed(key, self.hostname, now=now)
profile = None
else:
client = self._acme()
client.ensure_account() # a new-account limit (per IP) ends here, before anything was reserved
number, key = self.store.new_generation()
spki = spki_sha256(key.public_key())
if self.mode == "dns-01":
answer = self.broker.reserve(spki, "renewal" if state.get("confirmed_once") else "first")
self.clock.observe_if_stale(answer.server_time, CLOCK_SYNC_EVERY_S)
now = self.clock.now()
if isinstance(answer, Deferral):
self.store.discard(number)
state["blocked_until"], state["blocked_reason"] = min(answer.enroll_after, now + BLOCK_CEILING_S), "enroll_after"
self._save(state, active, now, "waiting", reason="enroll_after", wait_until=state["blocked_until"])
return PassResult("waiting", generation=active.number if active else None, wait_until=state["blocked_until"], reason="enroll_after")
reservation = answer
client.order_may_be_issued = False
state["attempt_count"] += 1
state["last_attempt"] = now
state["soft_failures"], state["soft_until"] = 0, None
self._persist(state) # counted BEFORE the order: a crash from here on still leaves the attempt on record
attempted = True
chain, profile = self._order(client, key, reservation, active, now)
self.store.hit("after_ca")
now = self.clock.now()
generation = self.store.write_chain(number, chain, self.hostname, {"ca": self._ca_tag, "profile": profile, "issued_at": int(now),
"reason": reason})
if not (generation.info.not_before - 600 <= now < generation.info.not_after):
raise TlsStoreError("chain_invalid", "the CA's certificate is not valid now")
# Written BEFORE the swap, so a crash on either side of it knows what to confirm and that the CA counted this issuance (the daily cap
# errs on the side of counting an issuance that never went into service).
issued = [t for t in state.get("issuances", []) if type(t) in (int, float) and now - DAY_S < t <= now]
state.update(issuances=[*issued, now], failures=0, blocked_until=None, blocked_reason=None, soft_until=None, soft_failures=0, last_error=None,
ari=None)
if self.mode == "dns-01":
state["pending_confirm"] = {"reservation_id": reservation.id, "spki": generation.spki, "gen": number,
"not_before": int(generation.info.not_before), "not_after": int(generation.info.not_after)}
self._persist(state)
self.store.hit("before_swap")
self.store.activate(number)
self.store.hit("after_swap")
except (AcmeError, BrokerError, TlsStoreError, httpx.HTTPError, OSError) as exc:
self._abandon(reservation, client)
return self._failed(state, active, exc, number, attempted)
except Exception as exc: # a bug must not take the process down, and must not repeat at once either
log.warning("certificate pass failed: %s", type(exc).__name__, exc_info=True)
self._abandon(reservation, client)
return self._failed(state, active, exc, number, attempted)
# from here the new certificate is serving: confirming and the status file only, nothing below may undo it
active = self._active()
self._flush_confirm(state, active)
self._save(state, active, now, "valid")
return PassResult("issued", generation=number)
def _abandon(self, reservation: Reservation | None, client: AcmeClient | None) -> None:
"""Tell the broker that the order of this reservation failed before the CSR was sent, so that the slot it keeps for an order that may have been issued is
freed (a validation that failed, a CA that was down). An order that got as far as the CSR is not abandoned: a certificate may exist, and only the CT
reconciliation can lift that hold. Best effort: a broker that cannot be reached leaves the slot to the reconciliation."""
if reservation is None or client is None or client.order_may_be_issued:
return
try:
self.broker.abandon(reservation.id)
except BrokerError as exc:
log.info("the abandoned order could not be reported (%s)", exc.word)
def _order(self, client: AcmeClient, key, reservation: Reservation | None, active: Generation | None, now: float) -> tuple[bytes, str | None]:
policy_c = self._policy
profile = client.offered_profile(policy_c["profile"], policy_c["profile_fallback"])
replaces = None
if active is not None and active.ca == self._ca_tag and now < active.info.not_after and "renewalInfo" in client.directory():
with contextlib.suppress(ValueError):
replaces = certificate_id(active.chain_pem)
csr = make_csr(key, self.hostname)
challenge = "dns-01" if self.mode == "dns-01" else "http-01"
publish, unpublish = (self._txt_hooks(reservation) if reservation is not None else (self.challenges.publish, self.challenges.unpublish))
while True:
try:
return client.obtain(self.hostname, csr, publish, unpublish, challenge=challenge, profile=profile, replaces=replaces), profile
except AcmeError as exc:
if replaces and (exc.type.endswith(":alreadyReplaced") or exc.status in (400, 403, 404, 409)
and not exc.type.endswith((":rateLimited", ":badNonce", ":invalidProfile", ":accountDoesNotExist"))):
# alreadyReplaced (409): another order already replaces that certificate. Boulder also REFUSES the whole order (it does not fall back to a
# plain one) when `replaces` is malformed, names a certificate it does not know, or one issued to another account (an account that was
# registered again): all of them renew as a plain order, once.
replaces = None
elif profile and exc.type.endswith(":invalidProfile"):
profile = None
elif exc.type.endswith(":accountDoesNotExist") and self._account_url():
# the CA no longer knows our account (a test CA was reset): forget it and register again, which needs the terms to be accepted
log.warning("the CA does not know the stored account any more: registering a new one")
self.store.write_file(self._account_files()[1], b"{}")
self._client_cache = None
client = self._acme()
else:
raise
def _txt_hooks(self, reservation: Reservation):
"""publish/unpublish for DNS-01 through the broker. The answer is CHECKED: the broker must say it wrote a TXT at _acme-challenge.<our host>;
anything else (a record of another name, another type) aborts the order and takes back what was written."""
expected = "_acme-challenge." + self.hostname
ttl = self._policy["txt_ttl_s"]
def publish(_token: str, proof: str) -> None:
if not SPKI_RE.fullmatch(proof):
raise BrokerError("bad_answer")
answer = self.broker.set_txt(reservation.id, proof)
if answer.name != expected or answer.type != "TXT" or answer.ttl != ttl:
raise BrokerError("dns_mismatch")
if self._dns_settle:
self._sleep(self._dns_settle)
def unpublish(_token: str) -> None:
try:
self.broker.clear_txt(reservation.id)
except BrokerError as exc: # the broker also clears it on confirm and when the reservation times out
log.info("the TXT could not be cleared now (%s)", exc.word)
return publish, unpublish
def _failed(self, state: dict[str, Any], active: Generation | None, exc: BaseException, number: int | None, attempted: bool) -> PassResult:
"""Whatever went wrong, the serving certificate was not touched. The generation that never served is removed, the failure is counted (when an
order had been started) and the next attempt waits: at least the gap, doubling, and as long as a rate limit or the broker asks."""
now = self.clock.now()
word = error_word(exc)
if number is not None:
with contextlib.suppress(TlsStoreError):
self.store.discard(number)
state["last_error"] = word
wait = 0.0
# Boulder: a rate limit is 429 rateLimited; the per-IP request limits of its load balancers are a bare 503 with Retry-After
rate_limited = isinstance(exc, AcmeError) and (exc.type.endswith(":rateLimited") or exc.status == 429
or (exc.status == 503 and exc.retry_after is not None))
if attempted:
state["failures"] = state.get("failures", 0) + 1
elif not rate_limited and not (isinstance(exc, BrokerError) and not exc.soft):
state["soft_failures"] = state.get("soft_failures", 0) + 1
state["soft_until"] = now + min(SOFT_RETRY_CEILING_S, SOFT_RETRY_FLOOR_S * 2 ** (state["soft_failures"] - 1))
retry = getattr(exc, "retry_after", None)
if rate_limited or (isinstance(exc, BrokerError) and (exc.status == 429 or exc.word in ("label_retired", "license_inactive"))):
# a limit (the CA's, a new-account-per-IP one, the broker's) is a wait, never a hard failure: the certificate in use keeps serving
wait = retry if retry else float(self._policy["min_attempt_gap_s"])
state["blocked_until"] = now + min(wait, BLOCK_CEILING_S)
state["blocked_reason"] = "rate_limited" if rate_limited else word
elif isinstance(exc, BrokerError) and not exc.soft and not attempted:
state["blocked_until"], state["blocked_reason"] = now + float(self._policy["min_attempt_gap_s"]), word
log.info("issuance did not complete (%s)", word)
waiting = self._wait(state, now)
until = waiting[0] if waiting else None
self._save(state, active, now, "waiting" if active is None or now < active.info.not_after else "expired", reason=word, wait_until=until)
return PassResult("failed", generation=active.number if active else None, error=word, wait_until=until)
# -- confirming and adopting
def _flush_confirm(self, state: dict[str, Any], active: Generation | None) -> None:
pending = state.get("pending_confirm")
if not pending or self.broker is None:
state.pop("pending_confirm", None)
return
if (not isinstance(pending, dict) or not isinstance(pending.get("reservation_id"), str) or not isinstance(pending.get("spki"), str)
or type(pending.get("not_before")) is not int or type(pending.get("not_after")) is not int):
pending = {} # a record that is not ours to read: dropped below with the ones that have nothing to confirm
if active is None or pending.get("gen") != active.number:
state.pop("pending_confirm", None) # the certificate it was for never went into service: there is nothing to confirm
self._persist(state)
return
try:
stamp = self.broker.confirm(pending["reservation_id"], pending["spki"], pending["not_before"], pending["not_after"])
self.clock.observe_if_stale(stamp, CLOCK_SYNC_EVERY_S)
except BrokerError as exc:
if exc.soft:
log.info("the confirmation will be sent later (%s)", exc.word)
return
log.warning("the broker refused the confirmation (%s)", exc.word) # an expired or closed reservation keeps its slot without it
state.pop("pending_confirm", None)
state["confirmed_once"] = True
self._persist(state)
def _adopt(self, now: float, state: dict[str, Any]) -> Generation | None:
"""A crash between "chain written" and "pointer swapped" leaves a complete, valid generation that nothing names. If this device made its key and
it is newer than the serving one, put it in service: the CA already counted it and the broker may already hold its reservation."""
if self.mode == "selfsigned":
return None
known = {row["spki_sha256"] for row in self.store.spki_log()}
active = self.store.active()
for generation in self.store.resumable(self.hostname):
if generation.ca != self._ca_tag or generation.spki not in known or generation.info.not_after <= now:
continue
if active is None or generation.info.not_before > active.info.not_before:
self.store.activate(generation.number)
log.info("generation %s was complete but not in service: now it is", generation.number)
return generation
break
return None
# -- the status file
def _status_document(self, active: Generation | None, state: dict[str, Any], now: float, word: str, *, reason: str = "", error: str = "",
wait_until: float | None = None) -> dict[str, Any]:
offset = self.clock.offset
try:
pointer = self.store.pointer()
except TlsStoreError:
pointer = None
ari = state.get("ari") or {}
document: dict[str, Any] = {
"v": 1, "state": word, "reason": reason or None, "mode": self.mode, "host": self.hostname, "ca": self._ca_tag,
"updated_at": int(now), "generation": active.number if active else None, "previous_generation": pointer.get("previous") if pointer else None,
"served": bool(pointer and pointer["served"]), "spki_sha256": active.spki if active else None,
"not_before": int(active.info.not_before) if active else None, "not_after": int(active.info.not_after) if active else None,
"profile": active.meta.get("profile") if active else None,
"attempts_total": state.get("attempt_count", 0), "failures": state.get("failures", 0),
"last_attempt_at": int(state["last_attempt"]) if type(state.get("last_attempt")) in (int, float) else None,
"next_attempt_at": int(wait_until) if wait_until else None,
"last_error": error or state.get("last_error") or None,
"clock_offset_s": round(offset, 1) if offset is not None else None,
"clock_skew": bool(offset is not None and abs(offset) > tls_policy()["authorization"]["node_skew_max_s"]),
"ari_window": [int(ari["start"]), int(ari["end"])] if "start" in ari else None,
"confirm_pending": bool(state.get("pending_confirm")),
}
return document
def _save(self, state: dict[str, Any], active: Generation | None, now: float, word: str, *, reason: str = "", wait_until: float | None = None) -> None:
self._persist(state)
status = self._status_document(active, state, now, word, reason=reason, wait_until=wait_until)
try:
status["key_storage"] = self.store.key_storage()
except TlsStoreError as exc:
status["key_storage"] = {"error": exc.word}
self.store.write_status(status)
def status(self) -> dict[str, Any] | None:
try:
return self.store.read_status()
except TlsStoreError:
return None
# -- what the 1.1 runner and the tunnel client already call
def current(self) -> CertState | None:
"""The serving certificate when it is usable now: it covers the name, comes from the configured CA and is within its validity."""
generation = self._active()
if generation is None or generation.ca != self._ca_tag or self.hostname not in generation.info.names:
return None
if not generation.info.not_before - 600 <= self.clock.now() < generation.info.not_after:
return None
return CertState(generation.chain_path, generation.key_path, generation.info, self._ca_tag, False, generation.number)
def due(self, state: CertState | None = None) -> bool:
return self._due_reason(self._active(), self.clock.now(), self._load_state_quiet()) is not None
def _load_state_quiet(self) -> dict[str, Any]:
try:
return self.store.read_state() or {}
except TlsStoreError:
return {}
def ensure(self, *, force: bool = False) -> CertState:
result = self.run_once(force=force)
state = self.current()
if state is None:
wait = max(0.0, (result.wait_until or 0.0) - self.clock.now()) if result.wait_until else None
raise CertUnavailable(f"no certificate yet ({result.reason or result.error or result.action})", type=result.error, retry_after=wait)
return replace(state, issued_now=result.issued)
def issue(self) -> CertState:
return self.ensure(force=True)
def next_wake(self) -> float:
"""Seconds until a pass can change anything: the earliest of the renewal moment, the next ARI check, the end of a wait, a confirmation to
retry. Between 60 s and an hour, so a changed clock or a changed file is noticed."""
now = self.clock.now()
state = self._load_state_quiet()
active = self._active()
moments = [now + 3600.0]
if active is not None:
moments.append(active.info.renew_at)
ari = state.get("ari") or {}
if ari.get("fp") == active.info.fingerprint:
moments.append(float(ari.get("next_check", now + 3600.0)))
if "start" in ari:
moments.append(ari["start"] + ari["frac"] * (ari["end"] - ari["start"]))
if state.get("pending_confirm"):
moments.append(now + 300.0)
wait = self._wait(state, now)
if wait is not None and (active is None or self._due_reason(active, now, state) is not None):
moments.append(wait[0])
return max(60.0, min(m for m in moments) - now) if len(moments) > 1 else 3600.0
# -- uninstall
def uninstall(self, *, revoke: bool = True) -> dict[str, Any]:
"""Plan section 4, in this order: the keys and certificates go FIRST, at once, with no network; then revokeCert, signed with the ACME account key
(read into memory before the folder was removed), for every certificate that was still in service; best effort. Releasing the label and the TXT
at the broker is the installer's next step (HttpBroker.release("uninstall")), after this one."""
evidence: dict[str, Any] = {"wiped": False, "revoked": 0, "revoke_errors": []}
leaves: list[bytes] = []
account_key = account_url = None
with contextlib.suppress(TlsStoreError):
for number in self.store.generations():
generation = self.store.generation(number)
if generation is not None and generation.ca == self._ca_tag:
leaves.append(leaf_der(generation.chain_pem))
raw = self.store.read_file(self._account_files()[0])
account_key = serialization.load_pem_private_key(raw, password=None) if raw else None
account_url = self._account_url()
self.store.destroy()
evidence["wiped"] = not self.dir.exists() or not any(self.dir.iterdir())
if not revoke or self.mode == "selfsigned" or not leaves or account_key is None or not account_url:
return evidence
client = AcmeClient(self.directory_url, account_key, http=self._http, account_url=account_url, sleep=self._sleep)
for der in leaves:
try:
client.revoke(der)
evidence["revoked"] += 1
except (AcmeError, httpx.HTTPError) as exc:
evidence["revoke_errors"].append(error_word(exc))
return evidence
async def renew_then_connect(manager: DeviceCertManager, connect: Callable[[], Any], *, wait: float | None = None) -> PassResult:
"""The order a device starts in: the certificate FIRST, the tunnel SECOND. A computer that was off for 100 days wakes with an expired certificate; the
tunnel can be up in a second, but ChatGPT would meet a certificate that fails, so the pass runs first (at most ``wait`` seconds, by default
policy.certificates.wait_after_install_s) and the tunnel connects afterwards whatever the pass said: a certificate that is not due, one that is
waiting for a rate limit and one that could not be issued still leave the connection worth making (the control channel is not the certificate)."""
limit = float(tls_policy()["certificates"]["wait_after_install_s"]) if wait is None else wait
try:
result = await asyncio.wait_for(asyncio.to_thread(manager.run_once), limit)
except asyncio.TimeoutError:
result = PassResult("failed", error="pass_timeout")
outcome = connect()
if asyncio.iscoroutine(outcome):
await outcome
return result
files/src/all_mcp/e2e_ct_watch.py 554 lines · 30.4 KB
"""ALL 2.0 WP36: the device watches Certificate Transparency for ITS OWN name (plan section 4, "Mala emisión detectable, con ancla en el dispositivo").
The two other detectors (the site Worker's cron and the independent ct_watch.py) compare CT with what the control plane's registry says, so a compromised control plane
could rewrite the registry and keep them quiet. This one cannot be rewritten from there: the only list it trusts is ``tls/spki-log`` (tls_store.py), one line for every key
THIS computer ever made, written before the key is used and never changed. Once a day, and ten minutes after each issuance, it asks CT what certificates exist for this
computer's own name and raises an alert for
unknown_key a certificate for the name whose key this computer never made (nobody else can have one honestly: an ACME order proves the key with a signature), and
bad_issuer a certificate for the name from anyone but Let's Encrypt, whatever its key.
Everything else is silent: our own certificates, and their precertificates (the same key and the same issuance: CT lists the pair, Cert Spotter as one entry and crt.sh as
two), so the pair is never counted twice and never alarms. The comparison does not look at serials, dates or the control plane.
What is asked, and of whom. Cert Spotter first (``/v1/issuances`` for the exact name, wildcards that cover it included), crt.sh when Cert Spotter cannot be read whole. The
request holds the name and fixed words and nothing else: no key, no licence, no device id, no cookie, no body; a redirect is refused. The name is public anyway (it is in CT);
the zone is never asked about alone, so a wildcard issued for the zone shows here only through Cert Spotter's ``match_wildcards`` (the other two detectors watch the zone).
Whole history, every time, never a cursor: a computer that was off for 100 days reads the certificates issued meanwhile (expired ones too) in its first look, and a source
that repeats, reorders or duplicates entries changes nothing.
Honesty about being blind. When this computer serves a certificate that is more than two days old, a source that is read whole must show it (it is the canary: a page that came
back empty or an index that is behind is not an all-clear). A source that answers with a page that does not parse whole (truncated, HTML, a row without a key hash), refuses, times out or hands out
more than this device is willing to read counts as down, and so does a crt.sh listing that needs more downloads than one run allows (the ones done are kept: the next
run finishes). Neither source whole for ``certificates.ct_degraded_after_s`` (72 h) is ``monitoring: degraded``, and the tray says so; a watch that never looked is
``pending``, and only a complete look that is not older than two intervals is ``ok``. Nothing here ever turns an unread CT into "all clear".
State, in ``tls/ct-watch.json`` (0600 in the 0700 folder, so uninstall wipes it with the keys): the alerts (kind, key hash, first seen, acknowledged), the moments of the last
look, and what crt.sh downloads already told. It holds no secret. ``ct_watch_status()`` is the shape ``e2e_status.build(ct=...)`` writes into ``e2e-status.json``: ``alert`` is
the tray's persistent message (an alert nobody acknowledged: ``acknowledge()``), ``alert_open`` is the fleet block's ``ct_alert`` (also true after an acknowledgement for as
long as the certificate that raised it is still valid: a person who saw it has not made it go away).
Timing comes from contracts/policy.json (``certificates.ct_check_interval_s``, ``ct_degraded_after_s``); this file only holds the pacing of its own retries.
"""
from __future__ import annotations
import contextlib
import hashlib
import json
import logging
import random
import re
import time
import urllib.parse
from dataclasses import dataclass, field
from typing import Any, Callable
import httpx
from cryptography import x509
from cryptography.hazmat.primitives import hashes, serialization
from . import e2e_policy as policy
from .tls_store import TlsStore, TlsStoreError, b64url
log = logging.getLogger("all_mcp.e2e_ct_watch")
STATE_FILE = "ct-watch.json"
LOCK_FILE = "ct-watch.lock"
SCHEMA_VERSION = 1
CERTSPOTTER_URL = "https://api.certspotter.com/v1/issuances"
CRTSH_URL = "https://crt.sh/"
USER_AGENT = "all-mcp-ct-selfwatch/2"
KINDS = ("unknown_key", "bad_issuer")
SOURCES = ("certspotter", "crtsh")
CANARY_AFTER_S = 172800.0 # a certificate older than this is expected to show in every CT source
CHECK_AFTER_ISSUANCE_S = 600.0 # a new certificate has reached the logs by now (the precertificate went in before the CA issued)
FETCH_TIMEOUT_S = 20.0
JSON_MAX_BYTES = 4 * 1024 * 1024 # a page of CT rows for one name; more than this is not read
PEM_MAX_BYTES = 64 * 1024
CERTSPOTTER_MAX_PAGES = 20
CRTSH_DOWNLOADS_PER_RUN = 12
RETRY_FIRST_S = 900.0 # a look that failed is retried after 15 min, 1 h, 4 h, 16 h, and then at the usual interval
RETRY_FACTOR = 4.0
INTERVAL_JITTER = 0.1 # a device looks between 0 and 10 % after the interval, so the fleet does not ask a source at the same moment
STALE_INTERVALS = 2 # a complete look older than this many intervals is no longer "ok"
ALERTS_KEPT = 64
CRTSH_KEPT = 256
_WORD = re.compile(r"[a-z0-9_]{1,48}")
_SPKI = re.compile(r"[A-Za-z0-9_-]{43}")
_ROW_ID = re.compile(r"[A-Za-z0-9_.-]{1,64}")
_CRTSH_ID = re.compile(r"[0-9]{1,20}")
_HEX_SHA256 = re.compile(r"[0-9a-f]{64}")
_LETS_ENCRYPT = re.compile(r"let['’]s encrypt", re.IGNORECASE)
class CtSourceError(Exception):
"""A source of CT that could not be read whole. ``word`` is a closed word (never text of the network) the state and the status may carry."""
def __init__(self, word: str):
super().__init__(word)
self.word = word
# ------------------------------------------------------------------------------------------------------------------------ certificates and the comparison
@dataclass(frozen=True)
class CtCert:
"""A certificate reduced to what the comparison needs, from whichever source. ``spki`` is the base64url SHA-256 of the SubjectPublicKeyInfo, the form of ``spki-log``."""
key: str
names: tuple[str, ...]
spki: str
issuer: str
not_after: int | None
source: str
@dataclass(frozen=True)
class Finding:
kind: str
spki: str
not_after: int | None
@property
def key(self) -> str:
return f"{self.kind}:{self.spki}"
def normalize(name: Any) -> str:
return name.strip().lower().rstrip(".") if isinstance(name, str) else ""
def covers(names: tuple[str, ...] | list[str], host: str) -> bool:
"""Whether a certificate with these names is one for ``host``: the name itself, or a wildcard that covers it (``*.zone`` covers exactly one label)."""
for name in names:
if name == host:
return True
if name.startswith("*.") and "." in host and host.split(".", 1)[1] == name[2:]:
return True
return False
def is_lets_encrypt(issuer: str) -> bool:
return _LETS_ENCRYPT.search(issuer) is not None
def evaluate(host: str, certs: list[CtCert], known: set[str]) -> list[Finding]:
"""What is wrong among the certificates CT showed for ``host``: against the keys THIS computer made (``known``), and against the issuer. One finding per (kind, key)."""
found: dict[str, Finding] = {}
for cert in certs:
if not covers(cert.names, host):
continue
if cert.spki not in known:
finding = Finding("unknown_key", cert.spki, cert.not_after)
found.setdefault(finding.key, finding)
if not is_lets_encrypt(cert.issuer):
finding = Finding("bad_issuer", cert.spki, cert.not_after)
found.setdefault(finding.key, finding)
return list(found.values())
def _seconds(text: Any) -> int | None:
if not isinstance(text, str):
return None
import datetime
candidate = text.strip().replace(" ", "T")
candidate = candidate if re.search(r"(Z|[+-]\d\d:?\d\d)$", candidate) else candidate + "Z"
try:
return int(datetime.datetime.fromisoformat(candidate.replace("Z", "+00:00")).timestamp())
except ValueError:
return None
def certs_from_certspotter(rows: Any, host: str) -> list[CtCert]:
"""Cert Spotter ``/v1/issuances`` rows (expand=dns_names, issuer). ``pubkey_sha256`` is the hex digest of the SPKI. A row that cannot be compared (no id, no key hash) makes
the page unusable: skipping it would be a certificate nobody looked at."""
if not isinstance(rows, list):
raise CtSourceError("bad_body")
certs = []
for row in rows:
if not isinstance(row, dict) or not isinstance(row.get("id"), str) or _ROW_ID.fullmatch(row["id"]) is None:
raise CtSourceError("bad_body")
digest = str(row.get("pubkey_sha256", "")).lower()
if _HEX_SHA256.fullmatch(digest) is None:
raise CtSourceError("bad_body")
issuer = row.get("issuer") if isinstance(row.get("issuer"), dict) else {}
names = tuple(normalize(name) for name in row["dns_names"] if isinstance(name, str)) if isinstance(row.get("dns_names"), list) else (host,) # asked for this name: no list, no doubt
certs.append(CtCert(key=str(row.get("tbs_sha256") or row["id"]), names=names, spki=b64url(bytes.fromhex(digest)),
issuer=" ".join(str(part) for part in (issuer.get("friendly_name"), issuer.get("name")) if isinstance(part, str)),
not_after=_seconds(row.get("not_after")), source="certspotter"))
return certs
def crtsh_groups(rows: Any) -> list[dict[str, Any]]:
"""crt.sh lists an issuance twice, as a precertificate and as the final certificate: same serial, same issuer, the same key. One row of each pair is enough (the rows of one
group are ``ids``, so a pair already read is known by either id)."""
if not isinstance(rows, list):
raise CtSourceError("bad_body")
groups: dict[str, dict[str, Any]] = {}
for row in rows:
if not isinstance(row, dict) or _CRTSH_ID.fullmatch(str(row.get("id", ""))) is None:
raise CtSourceError("bad_body")
serial = str(row.get("serial_number", "")).lower()
identity = f"{serial}|{row.get('issuer_ca_id')}" if serial else f"id|{row['id']}"
group = groups.setdefault(identity, {"ids": []})
group["ids"].append(str(row["id"]))
return list(groups.values())
def read_certificate(pem: bytes) -> tuple[tuple[str, ...], str, str, int | None]:
"""(names, SPKI hash, issuer, not_after) of a PEM certificate or precertificate. A key this cannot hash is not hidden: its hash is of the whole certificate, which no key of
this computer's list can equal, so it is an unknown key."""
try:
certificate = x509.load_pem_x509_certificate(pem)
except ValueError:
raise CtSourceError("bad_certificate") from None
try:
names = tuple(certificate.extensions.get_extension_for_class(x509.SubjectAlternativeName).value.get_values_for_type(x509.DNSName))
except x509.ExtensionNotFound:
names = ()
if not names:
names = tuple(attribute.value for attribute in certificate.subject.get_attributes_for_oid(x509.NameOID.COMMON_NAME) if isinstance(attribute.value, str))
try:
spki = b64url(hashlib.sha256(certificate.public_key().public_bytes(serialization.Encoding.DER, serialization.PublicFormat.SubjectPublicKeyInfo)).digest())
except Exception: # noqa: BLE001 - an algorithm this library does not know
spki = b64url(hashlib.sha256(b"unreadable-key|" + certificate.fingerprint(hashes.SHA256())).digest())
try:
not_after: int | None = int(certificate.not_valid_after_utc.timestamp())
except AttributeError: # cryptography < 42
import datetime
not_after = int(certificate.not_valid_after.replace(tzinfo=datetime.timezone.utc).timestamp())
return tuple(normalize(name) for name in names), spki, certificate.issuer.rfc4514_string(), not_after
# ------------------------------------------------------------------------------------------------------------------------ the two sources
class CtSources:
"""The questions, and nothing else. Every request is a GET to one of two fixed addresses with the device's own name (or a crt.sh certificate number) in the query and the
two fixed headers below; the name is checked against the contract's pattern first. Blocking: run it in a thread."""
def __init__(self, host: str, *, transport: httpx.BaseTransport | None = None):
if policy.anchored("identity.fqdn_regex").fullmatch(host) is None:
raise ValueError("not a device name")
self.host = host
self._http = httpx.Client(timeout=httpx.Timeout(FETCH_TIMEOUT_S), headers={"User-Agent": USER_AGENT, "Accept": "application/json"},
follow_redirects=False, trust_env=False, transport=transport)
def _get(self, url: str, limit: int) -> bytes:
try:
with self._http.stream("GET", url) as response:
status = response.status_code
if status == 429:
raise CtSourceError("rate_limited")
if status != 200:
raise CtSourceError("redirect" if 300 <= status < 400 else f"http_{status // 100}xx")
body = bytearray()
for chunk in response.iter_bytes():
body.extend(chunk)
if len(body) > limit:
raise CtSourceError("too_large")
return bytes(body)
except httpx.TimeoutException:
raise CtSourceError("timeout") from None
except httpx.HTTPError:
raise CtSourceError("unreachable") from None
def _json(self, url: str) -> Any:
try:
return json.loads(self._get(url, JSON_MAX_BYTES).decode("utf-8"))
except (ValueError, UnicodeError):
raise CtSourceError("bad_body") from None # a page cut short, an HTML error page
def certspotter(self) -> list[CtCert]:
"""Every issuance Cert Spotter knows for this name, from the first, page after page until an empty one."""
certs: dict[str, CtCert] = {}
after = ""
for _ in range(CERTSPOTTER_MAX_PAGES):
query = [("domain", self.host), ("include_subdomains", "false"), ("match_wildcards", "true"), ("expand", "dns_names"), ("expand", "issuer")]
if after:
query.append(("after", after))
rows = self._json(f"{CERTSPOTTER_URL}?{urllib.parse.urlencode(query)}")
if not isinstance(rows, list):
raise CtSourceError("bad_body")
if not rows:
return list(certs.values())
fresh = [cert for cert in certs_from_certspotter(rows, self.host) if cert.key not in certs]
if not fresh:
return list(certs.values()) # a server that ignores `after` and repeats a page has nothing more to say
certs.update((cert.key, cert) for cert in fresh)
after = rows[-1]["id"]
raise CtSourceError("too_many_pages") # never compare against part of the history
def crtsh(self, cache: dict[str, dict[str, Any]]) -> tuple[list[CtCert], dict[str, dict[str, Any]], bool]:
"""(certificates, what was downloaded this time, complete). ``cache`` is what earlier runs downloaded, by crt.sh id: only the groups not in it cost a request."""
groups = crtsh_groups(self._json(f"{CRTSH_URL}?{urllib.parse.urlencode([('q', self.host), ('output', 'json')])}"))
certs: list[CtCert] = []
fresh: dict[str, dict[str, Any]] = {}
downloads = 0
complete = True
for group in groups:
entry = next((cache[i] for i in group["ids"] if i in cache), None)
if entry is None:
if downloads >= CRTSH_DOWNLOADS_PER_RUN:
complete = False
continue
downloads += 1
names, spki, issuer, not_after = read_certificate(self._get(f"{CRTSH_URL}?{urllib.parse.urlencode([('d', group['ids'][0])])}", PEM_MAX_BYTES))
entry = {"names": list(names), "spki": spki, "issuer": issuer, "na": not_after}
fresh[group["ids"][0]] = entry
certs.append(CtCert(key=group["ids"][0], names=tuple(entry["names"]), spki=entry["spki"], issuer=entry["issuer"], not_after=entry["na"], source="crtsh"))
return certs, fresh, complete
# ------------------------------------------------------------------------------------------------------------------------ the state
def word(value: Any, default: str = "") -> str:
return value if isinstance(value, str) and _WORD.fullmatch(value) else default
def _int(value: Any) -> int | None:
return value if type(value) is int and 0 <= value < 10 ** 11 else None
def host_tag(host: str) -> str:
return hashlib.sha256(b"all-ct-selfwatch-v1/" + host.encode("ascii", "replace")).hexdigest()[:16]
def empty_state(tag: str, now: float) -> dict[str, Any]:
return {"v": SCHEMA_VERSION, "host": tag, "started_at": int(now), "next_check_at": None, "last_attempt_at": None, "last_ok_at": None, "fail_since": None,
"failures": 0, "source": "", "last_error": "", "alerts": {}, "crtsh": {}}
def clean_state(document: Any, tag: str | None, now: float) -> dict[str, Any]:
"""The state as read, shaped (a value of the wrong type is the default, nothing else is kept). A state of another name, or one that is not a state, starts again: CT is
read whole every time, so what it showed is raised again by the next look."""
if not isinstance(document, dict) or document.get("v") != SCHEMA_VERSION or (tag is not None and document.get("host") != tag):
return empty_state(tag or "", now)
state = empty_state(str(document.get("host", "")), now)
for key in ("started_at", "next_check_at", "last_attempt_at", "last_ok_at", "fail_since"):
state[key] = _int(document.get(key))
state["started_at"] = state["started_at"] if state["started_at"] is not None else int(now)
state["failures"] = min(_int(document.get("failures")) or 0, 1000)
state["source"] = word(document.get("source")) if document.get("source") in SOURCES else ""
state["last_error"] = word(document.get("last_error"))
alerts = document.get("alerts")
for key, alert in (alerts.items() if isinstance(alerts, dict) else ()):
if (isinstance(alert, dict) and alert.get("kind") in KINDS and isinstance(alert.get("spki"), str) and _SPKI.fullmatch(alert["spki"])
and key == f"{alert['kind']}:{alert['spki']}" and _int(alert.get("raised_at")) is not None):
state["alerts"][key] = {"kind": alert["kind"], "spki": alert["spki"], "raised_at": alert["raised_at"], "ack_at": _int(alert.get("ack_at")),
"not_after": _int(alert.get("not_after"))}
cache = document.get("crtsh")
for key, entry in (cache.items() if isinstance(cache, dict) else ()):
if (_CRTSH_ID.fullmatch(str(key)) and isinstance(entry, dict) and isinstance(entry.get("spki"), str) and isinstance(entry.get("issuer"), str)
and isinstance(entry.get("names"), list) and all(isinstance(name, str) for name in entry["names"])):
state["crtsh"][str(key)] = {"names": entry["names"][:100], "spki": entry["spki"][:64], "issuer": entry["issuer"][:300], "na": _int(entry.get("na"))}
return state
def monitoring(state: dict[str, Any], now: float) -> str:
"""``degraded`` once the looks have failed for the policy's 72 h, ``ok`` only for a complete look that is recent and not followed by a failure, ``pending`` otherwise
(never looked, a catch-up after a long sleep, a retry after one failure). Never ``ok`` by default."""
degraded_after = policy.value("certificates.ct_degraded_after_s")
interval = policy.value("certificates.ct_check_interval_s")
if state["fail_since"] is not None and now - state["fail_since"] >= degraded_after:
return "degraded"
if state["last_ok_at"] is not None and state["fail_since"] is None and now - state["last_ok_at"] <= STALE_INTERVALS * interval:
return "ok"
return "pending"
def summarize(state: dict[str, Any], now: float) -> dict[str, Any]:
"""The block ``e2e_status.build(ct=...)`` writes. ``alert``: some alert is not acknowledged. ``alert_open``: some alert is not acknowledged or its certificate is still
valid (or its end is not known): the fleet block's ``ct_alert``."""
alerts = list(state["alerts"].values())
waiting = [alert for alert in alerts if alert["ack_at"] is None]
open_ = [alert for alert in alerts if alert["ack_at"] is None or alert["not_after"] is None or alert["not_after"] > now]
return {"monitoring": monitoring(state, now), "alert": bool(waiting), "alert_open": bool(open_), "kinds": sorted({alert["kind"] for alert in open_}),
"count": len(alerts), "unacknowledged": len(waiting), "since": min((alert["raised_at"] for alert in waiting), default=None),
"last_ok_at": state["last_ok_at"], "last_check_at": state["last_attempt_at"], "next_check_at": state["next_check_at"], "fail_since": state["fail_since"],
"source": state["source"], "last_error": state["last_error"]}
def _unavailable(error: str) -> dict[str, Any]:
"""The block when the state cannot be read (an unsafe folder, a full disk): not green, not silent."""
return {"monitoring": "degraded", "alert": False, "alert_open": False, "kinds": [], "count": 0, "unacknowledged": 0, "since": None, "last_ok_at": None,
"last_check_at": None, "next_check_at": None, "fail_since": None, "source": "", "last_error": word(error, "storage_error")}
@dataclass
class CheckResult:
"""What one look did. ``ok`` is a complete comparison; ``new`` are the alerts raised by THIS look (never the ones already standing)."""
ok: bool
source: str = ""
error: str = ""
seen: int = 0
new: list[Finding] = field(default_factory=list)
# ------------------------------------------------------------------------------------------------------------------------ the watch
class CtWatch:
def __init__(self, host: str, *, store: TlsStore | None = None, sources: CtSources | None = None, clock: Callable[[], float] = time.time,
jitter: Callable[[], float] = random.random):
self.host = host
self.tag = host_tag(host)
self.store = store or TlsStore()
self.sources = sources or CtSources(host)
self.clock = clock
self.jitter = jitter
# -- the state
def _load(self) -> dict[str, Any]:
raw = self.store.read_file(STATE_FILE)
try:
document = json.loads(raw.decode("utf-8")) if raw else None
except (ValueError, UnicodeError):
document = None
return clean_state(document, self.tag, self.clock())
def _update(self, change: Callable[[dict[str, Any]], Any]) -> Any:
with self.store.locked(LOCK_FILE):
state = self._load()
outcome = change(state)
self.store.write_file(STATE_FILE, json.dumps(state, sort_keys=True, separators=(",", ":")).encode("utf-8"))
return outcome
def status(self) -> dict[str, Any]:
"""The block for ``e2e-status.json``; one cheap read of the state file."""
try:
return summarize(self._load(), self.clock())
except TlsStoreError as error:
return _unavailable(error.word)
def due_in(self) -> float:
"""Seconds until the next look (0 when it is due, or when the state cannot be read: the look itself says why)."""
try:
state = self._load()
except TlsStoreError:
return 0.0
now = self.clock()
interval = float(policy.value("certificates.ct_check_interval_s"))
due = state["next_check_at"]
if due is None:
return 0.0
return max(0.0, min(due - now, interval * (1 + INTERVAL_JITTER))) # a clock that went back cannot push the look further than one interval
def nudge(self, after_s: float = CHECK_AFTER_ISSUANCE_S) -> None:
"""A certificate was just issued: look in ``after_s`` seconds unless a look is due sooner."""
def change(state: dict[str, Any]) -> None:
soon = int(self.clock() + after_s)
state["next_check_at"] = soon if state["next_check_at"] is None else min(state["next_check_at"], soon)
with contextlib.suppress(TlsStoreError, OSError):
self._update(change)
def acknowledge(self) -> int:
return acknowledge(self.store, self.clock)
def _canary(self) -> str | None:
"""The key of the certificate this computer serves, once it is old enough to be in CT everywhere: a source that is read whole shows it. One that does not has an
incomplete view (a page that came back empty, an index that is behind), and an empty or short answer is never taken for an all-clear."""
try:
active = self.store.active()
except TlsStoreError:
return None
if active is None or self.clock() - active.info.not_before < CANARY_AFTER_S:
return None
return active.spki
# -- one look
def check(self) -> CheckResult:
"""Ask CT for this name and compare. Blocking. Never raises for what a source or the disk does: the outcome is in the result and in the state."""
try:
known = {row["spki_sha256"] for row in self.store.spki_log() if _SPKI.fullmatch(str(row.get("spki_sha256", "")))}
cache = self._load()["crtsh"]
except TlsStoreError as error:
return CheckResult(False, error=word(error.word, "storage_error"))
canary = self._canary()
certs: list[CtCert] = []
fresh: dict[str, dict[str, Any]] = {}
ok, source, errors = False, "", []
for name in SOURCES:
try:
if name == "certspotter":
got, complete = self.sources.certspotter(), True
else:
got, fresh, complete = self.sources.crtsh(cache)
except CtSourceError as error:
errors.append(error.word)
continue
except Exception as error: # noqa: BLE001 - a bug in a parser is a source that cannot be read, never a crashed watch
log.warning("CT source %s failed: %s", name, type(error).__name__)
errors.append("source_error")
continue
certs.extend(got) # what any source showed is compared, even when the look as a whole is not complete
if not complete:
errors.append("incomplete")
elif canary is not None and canary not in {cert.spki for cert in got}:
errors.append("canary_missing")
else:
ok, source = True, name
break
findings = evaluate(self.host, certs, known)
error = "" if ok else (errors[-1] if errors else "source_error")
result = CheckResult(ok, source, error, len(certs or []))
def record(state: dict[str, Any]) -> None:
now = int(self.clock())
interval = float(policy.value("certificates.ct_check_interval_s"))
for finding in findings:
if finding.key not in state["alerts"]:
state["alerts"][finding.key] = {"kind": finding.kind, "spki": finding.spki, "raised_at": now, "ack_at": None, "not_after": finding.not_after}
result.new.append(finding)
if len(state["alerts"]) > ALERTS_KEPT: # a flood of foreign certificates cannot grow the file: what nobody has seen stays, the oldest acknowledged go
ranked = sorted(state["alerts"].items(), key=lambda item: (item[1]["ack_at"] is None, item[1]["raised_at"]))
state["alerts"] = dict(ranked[-ALERTS_KEPT:])
state["crtsh"] = dict(list({**state["crtsh"], **fresh}.items())[-CRTSH_KEPT:])
state["last_attempt_at"] = now
if ok:
state.update(last_ok_at=now, fail_since=None, failures=0, source=source, last_error="")
state["next_check_at"] = int(now + interval * (1 + INTERVAL_JITTER * self.jitter()))
else:
state["failures"] = min(state["failures"] + 1, 1000)
state["fail_since"] = state["fail_since"] if state["fail_since"] is not None else now
state["last_error"] = word(error, "source_error")
state["next_check_at"] = int(now + min(interval, RETRY_FIRST_S * RETRY_FACTOR ** (state["failures"] - 1)))
try:
self._update(record)
except (TlsStoreError, OSError) as failure:
result.ok, result.error = False, word(getattr(failure, "word", ""), "storage_error") # the alerts of this look are in the result, the caller logs them
for finding in result.new:
log.error("CT shows a certificate for this computer's name that it did not make (%s)", finding.kind)
return result
def acknowledge(store: TlsStore | None = None, clock: Callable[[], float] = time.time) -> int:
"""The person has seen the alerts: the tray's message goes away until a NEW alert is raised. Returns how many were acknowledged. Callable from any process (the tray, a
command): the state is changed under its own lock."""
store = store or TlsStore()
with store.locked(LOCK_FILE):
raw = store.read_file(STATE_FILE)
try:
document = json.loads(raw.decode("utf-8")) if raw else None
except (ValueError, UnicodeError):
document = None
state = clean_state(document, None, clock())
count = 0
for alert in state["alerts"].values():
if alert["ack_at"] is None:
alert["ack_at"] = int(clock())
count += 1
if count:
store.write_file(STATE_FILE, json.dumps(state, sort_keys=True, separators=(",", ":")).encode("utf-8"))
return count
def stored_status(host: str, *, store: TlsStore | None = None, clock: Callable[[], float] = time.time) -> dict[str, Any] | None:
"""What the state file says without a watch running (the child is idle or the kill switch is on): None when there is no state, so nothing is said that was not seen.
An alert that stands is shown in every state of the encrypted path."""
store = store or TlsStore()
try:
raw = store.read_file(STATE_FILE)
document = json.loads(raw.decode("utf-8")) if raw else None
except (TlsStoreError, ValueError, UnicodeError):
return None
if not isinstance(document, dict) or document.get("host") != host_tag(host):
return None
return summarize(clean_state(document, host_tag(host), clock()), clock())
files/src/all_mcp/e2e_policy.json 261 lines · 6.5 KB
{
"$comment": "ALL 2.0 policy table: the ONE place where a number, a window or a limit of the end-to-end path is written down (plan section 0 and 7.1). The tunnel node (Go), the control plane (TypeScript) and the device (Python) read it; nobody copies a value into code (the contract generator lint fails the suite when one does). Units are in the key suffix: _s seconds, _bytes, _days, _pct percent. Keys listed in `provisional` are the plan's initial values: they are retuned after G0 and G1 and ONLY here. This is the device projection (the `copies` of the contract lint table): only the sections that runtime reads.",
"version": 1,
"provisional": [
"timing.stream_idle_s",
"timing.stream_max_age_s",
"timing.ping_interval_s",
"timing.dead_peer_s",
"node_limits.per_ip_concurrent",
"node_limits.per_ip_new_per_s",
"node_limits.openai_reserved_pct",
"terminator.endpoint_budgets",
"terminator.max_streams",
"migration.connected_now_s"
],
"protocol": {
"version": 2,
"alpn": "all-tunnel/2",
"line_max_bytes": 8192,
"open_line_max_bytes": 192,
"statement_line_max_bytes": 65552,
"jwt_max_bytes": 4096,
"first_line_timeout_s": 10,
"attach_timeout_s": 10,
"stream_id_bytes": 16,
"attach_secret_bytes": 32,
"nonce_bytes": 16,
"hint_max_addresses": 16,
"proof_htm": "GET",
"proof_htu_path": "/all-tunnel/2",
"node_assertion_header": "x-all-node-assertion",
"source_classes": [
"openai",
"hint",
"anon"
],
"close_reasons": [
"replaced",
"revoked",
"expired",
"released",
"killed",
"paused",
"drain",
"idle",
"protocol",
"shutdown"
],
"error_codes": [
"bad_hello",
"bad_command",
"unsupported_version",
"bad_proof",
"auth",
"clock_skew",
"unknown_stream",
"bad_attach",
"license_inactive",
"device_revoked",
"paused",
"busy",
"control_unavailable",
"line_too_long"
]
},
"timing": {
"ping_interval_s": 20,
"dead_peer_s": 45,
"stream_idle_s": 1800,
"stream_max_age_s": 14400,
"half_close_s": 30,
"backoff_min_s": 1,
"backoff_max_s": 60,
"renew_before_valid_until_s": 300,
"drain_quiet_s": 1200,
"feed_poll_s": 15,
"feed_apply_deadline_s": 20,
"reconnect_after_wake_s": 5
},
"authorization": {
"valid_until_paid_s": 14400,
"grace_paid_s": 900,
"access_ttl_paid_s": 14400,
"access_ttl_trial_s": 600,
"access_ttl_paywall_s": 600,
"access_future_slack_s": 60,
"node_skew_max_s": 30
},
"gated": {
"cutoff_days": 14
},
"identity": {
"issuer": "https://api.all-mcp.fyi",
"aud_tunnel": "all-tunnel",
"aud_access": "all-device-access",
"aud_node_assertion": "all-tunnel-control",
"label_length": 20,
"label_alphabet": "abcdefghijklmnopqrstuvwxyz234567",
"label_regex": "^[a-z2-7]{20}$",
"fqdn_regex": "^[a-z2-7]{20}\\.(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\\.)+[a-z]{2,63}$",
"fqdn_max_length": 253,
"tombstone_idle_months": 12,
"device_id_regex": "^[a-f0-9]{32}$",
"stream_id_regex": "^[0-9a-f]{32}$",
"attach_secret_regex": "^[A-Za-z0-9_-]{43}$",
"nonce_regex": "^[A-Za-z0-9_-]{22,64}$",
"jti_regex": "^[A-Za-z0-9_-]{16,128}$",
"kid_regex": "^[A-Za-z0-9._-]{1,128}$"
},
"node_limits": {
"per_ip_concurrent": 16,
"per_ip_new_per_s": 10,
"ipv6_prefix_bits": 64,
"openai_reserved_pct": 25,
"hint_ttl_s": 900
},
"terminator": {
"tls_handshake_s": 10,
"request_head_s": 10,
"request_head_max_bytes": 16384,
"max_streams": 64,
"endpoint_budgets": {
"register": {
"per_ip_per_min": 20
},
"authorize": {
"per_ip_per_min": 60
},
"token": {
"per_ip_per_min": 120
},
"pair": {
"per_ip_per_min": 30
},
"mcp_anonymous": {
"per_ip_per_min": 60
}
},
"tls_min_version": "TLSv1.3",
"tls_alpn": "http/1.1"
},
"surface": {
"body_max_bytes": 16777216,
"headers_max": 64,
"header_value_max_bytes": 32768,
"keep_alive_s": 75,
"limiter_lru_max": 4096,
"never_authorized_client_evict_hours": 24,
"identity_path": "/__all/surface/v1/proof",
"identity_challenge_header": "x-all-challenge",
"identity_mac_header": "x-all-challenge-mac",
"identity_proof_header": "x-all-surface-proof"
},
"pairing": {
"window_s": 600,
"code_ttl_e2e_s": 600,
"code_ttl_local_s": 86400,
"max_codes_local": 16,
"bad_codes_burn": 5
},
"certificates": {
"profile": "tlsserver",
"profile_fallback": "classic",
"profile_days": 45,
"renew_backstop_divisor": 3,
"ari_checks_per_day": 2,
"min_attempt_gap_s": 21600,
"max_issuances_per_day": 2,
"txt_ttl_s": 60,
"txt_value_length": 43,
"ct_check_interval_s": 86400,
"ct_degraded_after_s": 259200,
"wait_after_install_s": 300
},
"migration": {
"legacy_close_after_days": 30,
"delete_old_connector_after_days": 7,
"connected_now_s": 300,
"wave_hash_prefix": "all-e2e-wave-v1/",
"enroll_refresh_min_s": 900,
"enroll_refresh_max_s": 1800,
"kill_apply_deadline_s": 3600,
"blocked_after_failures": 3,
"blocked_retry_s": 21600,
"selfcheck": {
"timeout_s": 10,
"min_interval_s": 60
}
},
"keep_awake": {
"hold_cap_s": 28800,
"release_grace_s": 5,
"reconcile_s": 2
},
"enums": {
"authorize_status": [
"active",
"gated",
"paused"
],
"statement_mode": [
"active",
"paywall"
],
"budget_class": [
"paid",
"trial",
"flex"
],
"acme_purpose": [
"first",
"renewal"
],
"reserve_status": [
"granted",
"deferred"
],
"feed_event_type": [
"revoke",
"pause",
"resume",
"license_end",
"release",
"kill",
"kill_clear"
],
"e2e_state": [
"off",
"unenrolled",
"waiting",
"issuing",
"connecting",
"ready",
"degraded",
"clock_skew",
"compat_only",
"killed"
],
"e2e_cohort": [
"paid",
"trial",
"flex",
"legacy"
],
"ineligible_reason": [
"update_not_in_plan",
"legacy_credential",
"unsupported_version"
],
"selfcheck_reason": [
"chain",
"expired",
"spki",
"name",
"tls_version",
"healthz",
"unreachable",
"timeout"
]
}
}
files/src/all_mcp/e2e_policy.py 43 lines · 1.9 KB
"""The device's read of the ALL 2.0 policy table (contracts/policy.json, the contract).
The device payload holds only `src/`, so it cannot read `contracts/` at run time: `e2e_policy.json` next to this file is a byte-identical copy that
the generator's `sync` command writes and its `sync --check` compares (the contract test suite). Code never types a number or a pattern of the table
again: it asks for it by its dotted name, `value("surface.body_max_bytes")`, and the generator's lint refuses a copy.
"""
from __future__ import annotations
import functools
import json
import re
from pathlib import Path
from typing import Any
POLICY_FILE = Path(__file__).with_name("e2e_policy.json")
@functools.lru_cache(maxsize=1)
def load() -> dict[str, Any]:
return json.loads(POLICY_FILE.read_text(encoding="utf-8"))
def value(dotted: str) -> Any:
"""`value("protocol.line_max_bytes")`: the one way this device reads a number, a word or a list of the table."""
node: Any = load()
for part in dotted.split("."):
node = node[part]
return node
def anchored(dotted: str) -> re.Pattern[str]:
"""The table's `^...$` pattern as a compiled one to use with `fullmatch` (Python's `$` also matches before a final newline, so the anchors are
dropped, the way the contract verifiers do)."""
pattern = value(dotted)
if not (pattern.startswith("^") and pattern.endswith("$")):
raise ValueError("policy patterns are anchored: " + dotted)
return re.compile(pattern[1:-1])
def host_pattern(zone: str) -> re.Pattern[str]:
"""The detector this device builds for ITS OWN address in a zone: the table's label pattern, one dot, the zone. Anchored with \\A and \\Z, so
match, search and fullmatch give the verdict the Go and TypeScript detectors give."""
return re.compile(r"\A" + value("identity.label_regex")[1:-1] + r"\." + re.escape(zone) + r"\Z")
files/src/all_mcp/e2e_terminator.py 839 lines · 42.2 KB
"""The TLS terminator of ALL 2.0 (plan sections 2, 3 and 5): where ChatGPT's HTTPS ends on this computer.
The tunnel node forwards bytes it cannot read. Every stream the node opens for this computer (``OPEN``, then the device's ``ATTACH``) is a TLS connection that
ChatGPT made to ``<label>.<zone>``; this module wraps it in a server-side TLS with the device's own certificate and hands the plain HTTP/1.1 to the agent's E2E
listener, and nothing else:
* TLS 1.3 at least, ALPN ``http/1.1``, and the SNI must be this computer's own name (a handshake for any other name, or with none, ends with an alert);
the certificate is the one ``tls/current`` names at the moment of the handshake (``tls_store.ServingContext``), so a renewal drops no stream.
* The terminator sees the HTTP in clear and the client's address in the OPEN (the TLS it terminates hides it from the agent), so the limits that need an
address live here: streams in all, per address, per second, and per endpoint (``/register``, ``/authorize``, ``/token``, ``/pair``) per minute, with a part
of the streams kept for class ``openai`` so that a flood of strangers cannot starve ChatGPT. The agent never sees an address and never needs one.
* Nothing is rewritten. The request head is checked and forwarded byte for byte: it must arrive whole within ``terminator.request_head_s`` and within
``terminator.request_head_max_bytes``, ends at CRLFCRLF (a bare LF, an obsolete line fold, a repeated or non-numeric ``Content-Length``, a ``Content-Length``
together with ``Transfer-Encoding``, or an ``Upgrade`` is refused: h11 in the agent would unfold, merge or accept them, which is how a request is smuggled).
The body is followed by its framing (length or chunks), so that the NEXT request on the same connection is a head again and is counted too.
* The destination is the ONE address in ``surfaces.json`` (never LOCAL), read again for every stream, its pid alive, and before the first byte of ChatGPT's
goes anywhere the listener proves, on that same connection, that it is the agent (the HMAC challenge of surfaces.prove_listener): a port the agent freed
and another process bound receives the challenge and nothing more.
* Idle time and age come from the policy the node offered clamped to this device's own (``e2e_tunnel.Windows``); a stream is closed when it is silent for the
first or has lived the second.
No address is written to a log (the plan: nothing persistent names a client).
"""
from __future__ import annotations
import asyncio
import contextlib
import ipaddress
import logging
import re
import socket
import ssl
import time
from collections import OrderedDict, deque
from dataclasses import dataclass, field
from typing import Any, Callable
from . import e2e_policy as policy
from . import surfaces, tls_policy
log = logging.getLogger("all_mcp.e2e_terminator")
TLS_HANDSHAKE_BYTE = 0x16
# Constants of this module that the policy does not define (it defines the windows, the limits and the sizes): how long a local dial may take, how much of the
# agent's answer is read at once, and how the keepalive of a TCP connection is shaped.
LOCAL_DIAL_SECONDS = 5.0
READ_BYTES = 65536
KEEPALIVE_IDLE_SECONDS = 30
KEEPALIVE_INTERVAL_SECONDS = 10
KEEPALIVE_PROBES = 3
RATE_WINDOW_SECONDS = 1.0
BUDGET_WINDOW_SECONDS = 60.0
CHUNK_LINE_MAX = 64
TRAILER_MAX_BYTES = 16384
_TOKEN = re.compile(rb"[!#$%&'*+.^_`|~0-9A-Za-z-]+")
_REQUEST_LINE = re.compile(rb"([!#$%&'*+.^_`|~0-9A-Za-z-]{1,32}) (/[\x21-\x7e]*) HTTP/1\.([01])")
_DIGITS = re.compile(rb"[0-9]{1,18}")
_CHUNK_SIZE = re.compile(rb"[0-9a-fA-F]{1,15}")
# The end-points that are budgeted per address: the keys of policy.terminator.endpoint_budgets, which are also the paths the E2E surface answers, except
# ANONYMOUS_MCP, which is not a path: it is /mcp asked for without a credential the terminator has seen work (no bearer, or a bearer from an address that has
# never completed an authenticated call), the first call of every connector (the 401 that starts OAuth discovery) and what a scanner sends.
ANONYMOUS_MCP = "mcp_anonymous"
BUDGETED = tuple(name for name in policy.value("terminator.endpoint_budgets") if name != ANONYMOUS_MCP)
AUTHENTICATED_PATHS = frozenset({"/mcp", "/mcp/", "/token"})
# How long an address that completed an authenticated call is remembered as a client of this computer (ChatGPT's own addresses keep renewing it).
KNOWN_CLIENT_SECONDS = 3600.0
# ------------------------------------------------------------------------------------------------------------------------ small helpers
def ip_group(ip: str) -> str:
"""What a client address is counted as: an IPv4 address by itself, an IPv6 address by its /64 (a household or an ISP lease owns the whole /64), an IPv4-mapped
IPv6 address as the IPv4 it carries. Anything that is not an address is one shared group, so the limits still apply to it."""
try:
address = ipaddress.ip_address(ip)
except ValueError:
return "invalid"
if isinstance(address, ipaddress.IPv6Address):
if address.ipv4_mapped is not None:
return str(address.ipv4_mapped)
return str(ipaddress.ip_network((str(address), policy.value("node_limits.ipv6_prefix_bits")), strict=False))
return str(address)
def keepalive(writer: asyncio.StreamWriter) -> None:
"""TCP keepalive on the connection (a half-open one, after a network change, must not live for ever)."""
sock = writer.get_extra_info("socket")
if sock is None:
return
try:
sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
if hasattr(socket, "TCP_KEEPIDLE"):
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, KEEPALIVE_IDLE_SECONDS)
elif hasattr(socket, "TCP_KEEPALIVE"): # macOS
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPALIVE, KEEPALIVE_IDLE_SECONDS)
if hasattr(socket, "TCP_KEEPINTVL"):
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, KEEPALIVE_INTERVAL_SECONDS)
if hasattr(socket, "TCP_KEEPCNT"):
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, KEEPALIVE_PROBES)
except OSError:
pass
async def close_writer(writer: asyncio.StreamWriter | None) -> None:
if writer is None:
return
with contextlib.suppress(Exception):
writer.close()
with contextlib.suppress(Exception):
await asyncio.wait_for(writer.wait_closed(), 2.0)
def http_response(status: int, reason: str, text: str, *, extra: tuple[tuple[str, str], ...] = ()) -> bytes:
"""A complete small HTTP/1.1 answer the terminator gives itself (a refusal, never a rewrite of what the agent says): plain text, closes the connection."""
body = (text + "\n").encode("ascii")
lines = [f"HTTP/1.1 {status} {reason}", "Content-Type: text/plain", f"Content-Length: {len(body)}", "Cache-Control: no-store", "Connection: close"]
lines.extend(f"{name}: {value}" for name, value in extra)
return ("\r\n".join(lines) + "\r\n\r\n").encode("ascii") + body
# ------------------------------------------------------------------------------------------------------------------------ TLS over a stream that is not a socket
class TLSStream:
"""Server-side TLS over any asyncio byte stream (here: the data connection to the tunnel node, which is itself TLS). ``ssl.MemoryBIO`` lets TLS run over a stream
that is not a socket, which asyncio's own start_tls cannot do on top of another TLS transport."""
def __init__(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter, context: ssl.SSLContext, *, initial: bytes = b""):
self._reader = reader
self._writer = writer
self._incoming = ssl.MemoryBIO()
self._outgoing = ssl.MemoryBIO()
self._ssl = context.wrap_bio(self._incoming, self._outgoing, server_side=True)
if initial:
self._incoming.write(initial)
self._eof = False
self._decrypted = b"" # bytes already decrypted when a read was cancelled while it flushed a key update: given back by the next read, never lost
async def flush(self) -> None:
data = self._outgoing.read()
if data:
self._writer.write(data)
await self._writer.drain()
async def _feed(self) -> bool:
if self._eof:
return False
data = await self._reader.read(READ_BYTES)
if not data:
self._incoming.write_eof()
self._eof = True
return False
self._incoming.write(data)
return True
async def handshake(self, timeout: float) -> None:
async def go() -> None:
while True:
try:
self._ssl.do_handshake()
return
except ssl.SSLWantReadError:
await self.flush()
if not await self._feed():
raise ConnectionResetError("the client closed during the TLS handshake")
try:
await asyncio.wait_for(go(), timeout)
finally:
with contextlib.suppress(Exception):
await self.flush() # the final flight, or the alert that explains a refusal
def selected_alpn(self) -> str | None:
return self._ssl.selected_alpn_protocol()
def version(self) -> str | None:
return self._ssl.version()
async def read(self, size: int = READ_BYTES) -> bytes:
"""Decrypted bytes; b"" when the client closed (close_notify or a plain close). Safe to cancel (a timeout): nothing that was decrypted is lost."""
if self._decrypted:
data, self._decrypted = self._decrypted, b""
return data
while True:
try:
data = self._ssl.read(size)
except ssl.SSLWantReadError:
await self.flush()
if not await self._feed():
return b""
continue
except (ssl.SSLZeroReturnError, ssl.SSLEOFError):
return b""
if self._outgoing.pending:
self._decrypted = data
await self.flush() # TLS 1.3 key updates and the like
self._decrypted = b""
return data
async def write(self, data: bytes) -> None:
view = memoryview(data)
while view:
written = self._ssl.write(view[:16384])
view = view[written:]
if self._outgoing.pending >= READ_BYTES or not view:
await self.flush()
async def close(self) -> None:
"""Say close_notify and go; the peer's own close_notify is not waited for."""
with contextlib.suppress(ssl.SSLError, OSError, ValueError):
self._ssl.unwrap()
with contextlib.suppress(Exception):
await self.flush()
# ------------------------------------------------------------------------------------------------------------------------ limits
@dataclass(eq=False)
class Ticket:
"""One admitted stream: give it back with ``Admission.release``."""
group: str
cls: str
released: bool = False
class Admission:
"""Who may hold a stream, decided from the OPEN (the client's address and the node's class for it) before the device dials the node back.
The streams in all are ``terminator.max_streams``; ``node_limits.openai_reserved_pct`` of them are reserved to class ``openai`` (strangers and hinted
addresses together can hold only the rest, so a flood from anywhere never leaves ChatGPT without a slot). A stranger or hinted address is also held to
``per_ip_concurrent`` streams and ``per_ip_new_per_s`` new ones a second (an IPv6 address by its /64); class ``openai`` is not counted per address, because
a handful of addresses carry all of ChatGPT. Every table is bounded (``surface.limiter_lru_max``), the oldest entry is the first to go."""
def __init__(self, *, max_streams: int | None = None, reserved_pct: int | None = None, per_ip_concurrent: int | None = None,
per_ip_new_per_s: int | None = None, table_max: int | None = None, clock: Callable[[], float] = time.monotonic):
self.total_max = policy.value("terminator.max_streams") if max_streams is None else max_streams
percent = policy.value("node_limits.openai_reserved_pct") if reserved_pct is None else reserved_pct
self.reserved = -(-self.total_max * percent // 100) # rounded up: a quarter of 3 streams is still one stream
self.per_ip = policy.value("node_limits.per_ip_concurrent") if per_ip_concurrent is None else per_ip_concurrent
self.per_ip_rate = policy.value("node_limits.per_ip_new_per_s") if per_ip_new_per_s is None else per_ip_new_per_s
self._table_max = policy.value("surface.limiter_lru_max") if table_max is None else table_max
self._clock = clock
self.active = 0
self.active_other = 0
self._concurrent: dict[str, int] = {}
self._arrivals: OrderedDict[str, deque[float]] = OrderedDict()
@property
def other_max(self) -> int:
return self.total_max - self.reserved
def admit(self, ip: str, cls: str) -> tuple[Ticket | None, str]:
"""(a ticket, "") when the stream may go on, else (None, the word that says why)."""
group = ip_group(ip)
openai = cls == "openai"
if self.active >= self.total_max:
return None, "full"
if not openai:
if self.active_other >= self.other_max:
return None, "full_other"
if self._concurrent.get(group, 0) >= self.per_ip:
return None, "ip_concurrent"
now = self._clock()
window = self._arrivals.get(group)
if window is None:
window = self._arrivals[group] = deque()
while len(self._arrivals) > self._table_max:
self._arrivals.popitem(last=False)
else:
self._arrivals.move_to_end(group)
while window and now - window[0] >= RATE_WINDOW_SECONDS:
window.popleft()
if len(window) >= self.per_ip_rate:
return None, "ip_rate"
window.append(now)
self._concurrent[group] = self._concurrent.get(group, 0) + 1
self.active_other += 1
self.active += 1
return Ticket(group, cls), ""
def release(self, ticket: Ticket) -> None:
if ticket.released:
return
ticket.released = True
self.active -= 1
if ticket.cls != "openai":
self.active_other -= 1
left = self._concurrent.get(ticket.group, 1) - 1
if left > 0:
self._concurrent[ticket.group] = left
else:
self._concurrent.pop(ticket.group, None)
class EndpointBudgets:
"""The budgets per address and per endpoint class, per minute (``terminator.endpoint_budgets``): the endpoints that anybody who can read the certificate
transparency logs can reach without a token (``/register``, ``/authorize``, ``/token``, ``/pair``). One address using up its budget costs the others nothing."""
def __init__(self, *, budgets: dict[str, dict[str, int]] | None = None, table_max: int | None = None, clock: Callable[[], float] = time.monotonic):
self._budgets = policy.value("terminator.endpoint_budgets") if budgets is None else budgets
self._table_max = policy.value("surface.limiter_lru_max") if table_max is None else table_max
self._clock = clock
self._windows: OrderedDict[tuple[str, str], deque[float]] = OrderedDict()
self._known: OrderedDict[str, float] = OrderedDict() # address group -> when it last completed an authenticated call
@staticmethod
def classify(path: str, *, credential_seen: bool = True) -> str | None:
"""The budgeted endpoint a request path belongs to (its query is not part of it), or None. `/mcp` without a credential the terminator has seen work is the
anonymous class of the policy; with one, it is not budgeted (ChatGPT's addresses carry the traffic of every user of this computer)."""
name = path.split("?", 1)[0].rstrip("/")
if name == "/mcp" and not credential_seen:
return ANONYMOUS_MCP
return name[1:] if name[1:] in BUDGETED else None
def note_client(self, ip: str) -> None:
"""This address just completed an authenticated call: it is a client of this computer, and its `/mcp` calls are no longer counted as anonymous."""
key = ip_group(ip)
self._known[key] = self._clock()
self._known.move_to_end(key)
while len(self._known) > self._table_max:
self._known.popitem(last=False)
def is_client(self, ip: str) -> bool:
key = ip_group(ip)
since = self._known.get(key)
if since is None:
return False
if self._clock() - since >= KNOWN_CLIENT_SECONDS:
del self._known[key]
return False
return True
def allow(self, endpoint: str, ip: str) -> tuple[bool, float]:
"""(True, 0) and the request is counted, or (False, seconds until it would be allowed)."""
limit = self._budgets[endpoint]["per_ip_per_min"]
key = (endpoint, ip_group(ip))
now = self._clock()
window = self._windows.get(key)
if window is None:
window = self._windows[key] = deque()
while len(self._windows) > self._table_max:
self._windows.popitem(last=False)
else:
self._windows.move_to_end(key)
while window and now - window[0] >= BUDGET_WINDOW_SECONDS:
window.popleft()
if len(window) >= limit:
return False, max(1.0, BUDGET_WINDOW_SECONDS - (now - window[0]))
window.append(now)
return True, 0.0
# ------------------------------------------------------------------------------------------------------------------------ the request head and its framing
class BadRequest(Exception):
"""A request the terminator refuses. `status` is what the client is told (when it can still be told), `word` a fixed word for the counters."""
def __init__(self, status: int, word: str):
super().__init__(word)
self.status = status
self.word = word
@dataclass(frozen=True)
class RequestHead:
method: str
target: str
path: str
headers: tuple[tuple[bytes, bytes], ...] # lower-case name, value as sent (without the surrounding blanks)
body: str # "none" | "length" | "chunked"
length: int
raw_size: int
def header(self, name: bytes) -> bytes | None:
for key, value in self.headers:
if key == name:
return value
return None
def parse_head(raw: bytes) -> RequestHead:
"""Check one request head (the bytes up to and including the CRLFCRLF) and say how its body is framed. Raises BadRequest. Nothing is changed: the caller forwards `raw`."""
if len(raw) > policy.value("terminator.request_head_max_bytes"):
raise BadRequest(431, "head_too_large")
if not raw.endswith(b"\r\n\r\n"):
raise BadRequest(400, "bad_head")
body = raw[:-4]
if b"\n" in body.replace(b"\r\n", b"") or b"\r" in body.replace(b"\r\n", b""):
raise BadRequest(400, "bare_line_ending") # a lone LF or CR: what a lenient parser reads as a line end and a strict one does not
lines = body.split(b"\r\n")
found = _REQUEST_LINE.fullmatch(lines[0])
if found is None:
raise BadRequest(400, "bad_request_line")
headers: list[tuple[bytes, bytes]] = []
for line in lines[1:]:
if line[:1] in (b" ", b"\t"):
raise BadRequest(400, "obs_fold") # h11 unfolds an obsolete line fold into the previous header: the agent would read what this check never saw
name, colon, value = line.partition(b":")
if not colon or _TOKEN.fullmatch(name) is None:
raise BadRequest(400, "bad_header")
value = value.strip(b" \t")
if any(byte < 0x20 and byte != 0x09 or byte == 0x7F for byte in value):
raise BadRequest(400, "bad_header_value")
if len(value) > policy.value("surface.header_value_max_bytes"):
raise BadRequest(431, "header_too_large")
headers.append((name.lower(), value))
if len(headers) > policy.value("surface.headers_max"):
raise BadRequest(431, "too_many_headers")
lengths = [value for name, value in headers if name == b"content-length"]
encodings = [value for name, value in headers if name == b"transfer-encoding"]
if len(lengths) > 1:
raise BadRequest(400, "repeated_content_length") # even identical: h11 merges them, a proxy in front of another parser may not
if len(encodings) > 1 or (encodings and encodings[0].lower() != b"chunked") or (encodings and lengths):
raise BadRequest(400, "bad_framing")
for name, value in headers:
if name == b"upgrade" or (name == b"connection" and b"upgrade" in value.lower()):
raise BadRequest(400, "upgrade") # the E2E surface is plain HTTP: there is no protocol to switch to
length, kind = 0, "none"
if lengths:
if _DIGITS.fullmatch(lengths[0]) is None:
raise BadRequest(400, "bad_content_length")
length, kind = int(lengths[0]), "length"
if length > policy.value("surface.body_max_bytes"):
raise BadRequest(413, "body_too_large")
elif encodings:
kind = "chunked"
target = found.group(2).decode("ascii")
path = target.split("?", 1)[0]
if "%" in path:
raise BadRequest(400, "encoded_path") # the agent's server decodes the path before it routes (/%72egister is /register): the budgets below would read another name
return RequestHead(found.group(1).decode("ascii"), target, path, tuple(headers), kind, length, len(raw))
@dataclass
class Segment:
"""A slice of what the client sent, in order: a head (with its parse) or part of a body (or of its chunk framing). Together they are exactly the bytes received."""
kind: str # "head" | "body"
data: bytes
head: RequestHead | None = None
class RequestFramer:
"""Splits the client's byte stream into request heads and body, so that every request of a keep-alive connection (not only the first) is a head again. A head is
held back until it is whole and has passed ``parse_head`` (an incomplete head is never forwarded); body bytes are passed on as they arrive. Raises BadRequest."""
def __init__(self) -> None:
self._buffer = bytearray()
self._state = "head"
self._remaining = 0
self._total = 0
self._trailer = 0
self._chunked_body = False
@property
def awaiting_head(self) -> bool:
return self._state == "head"
@property
def partial_head(self) -> bool:
return self._state == "head" and bool(self._buffer)
def feed(self, data: bytes) -> list[Segment]:
self._buffer += data
out: list[Segment] = []
while self._buffer:
if self._state == "head":
end = self._buffer.find(b"\r\n\r\n")
if end < 0:
if len(self._buffer) > policy.value("terminator.request_head_max_bytes"):
raise BadRequest(431, "head_too_large")
break
raw = bytes(self._buffer[:end + 4])
head = parse_head(raw)
del self._buffer[:end + 4]
out.append(Segment("head", raw, head))
self._total = 0
if head.body == "length" and head.length:
self._state, self._remaining = "body", head.length
elif head.body == "chunked":
self._state = "chunk_size"
elif self._state == "body":
take = min(self._remaining, len(self._buffer))
out.append(Segment("body", bytes(self._buffer[:take])))
del self._buffer[:take]
self._remaining -= take
if not self._remaining:
self._state = "head"
elif self._state == "chunk_size":
end = self._buffer.find(b"\r\n")
if end < 0:
if len(self._buffer) > CHUNK_LINE_MAX:
raise BadRequest(400, "bad_chunk")
break
if end > CHUNK_LINE_MAX or _CHUNK_SIZE.fullmatch(bytes(self._buffer[:end])) is None:
raise BadRequest(400, "bad_chunk") # no chunk extensions: a size is hexadecimal digits and nothing else
size = int(self._buffer[:end], 16)
out.append(Segment("body", bytes(self._buffer[:end + 2])))
del self._buffer[:end + 2]
self._total += size
if self._total > policy.value("surface.body_max_bytes"):
raise BadRequest(413, "body_too_large")
self._state, self._remaining, self._trailer = ("trailer", 0, 0) if size == 0 else ("chunk_data", size, 0)
elif self._state == "chunk_data":
take = min(self._remaining, len(self._buffer))
out.append(Segment("body", bytes(self._buffer[:take])))
del self._buffer[:take]
self._remaining -= take
if not self._remaining:
self._state = "chunk_end"
elif self._state == "chunk_end":
if len(self._buffer) < 2:
break
if bytes(self._buffer[:2]) != b"\r\n":
raise BadRequest(400, "bad_chunk")
out.append(Segment("body", b"\r\n"))
del self._buffer[:2]
self._state = "chunk_size"
else: # trailer: header lines up to the empty one
end = self._buffer.find(b"\r\n")
if end < 0:
if len(self._buffer) + self._trailer > TRAILER_MAX_BYTES:
raise BadRequest(431, "head_too_large")
break
line = bytes(self._buffer[:end + 2])
self._trailer += len(line)
if self._trailer > TRAILER_MAX_BYTES or b"\n" in line[:-2] or line[:1] in (b" ", b"\t"):
raise BadRequest(400, "bad_chunk")
out.append(Segment("body", line))
del self._buffer[:end + 2]
if end == 0:
self._state = "head"
return out
# ------------------------------------------------------------------------------------------------------------------------ the terminator
@dataclass
class TerminatorStats:
streams: int = 0
active: int = 0
tls_streams: int = 0
handshake_failures: int = 0
no_certificate: int = 0
refused: dict[str, int] = field(default_factory=dict) # admission refusals, by word
bad_requests: dict[str, int] = field(default_factory=dict)
budget_refused: int = 0
no_surface: int = 0
identity_failures: int = 0
agent_unreachable: int = 0
head_timeouts: int = 0
idle_closes: int = 0
age_closes: int = 0
bytes_up: int = 0 # client -> agent
bytes_down: int = 0 # agent -> client
def as_dict(self) -> dict[str, Any]:
return dict(self.__dict__)
def count(self, table: dict[str, int], word: str) -> None:
table[word] = table.get(word, 0) + 1
class Terminator:
"""Serves the streams the node opens: TLS, the checks above, the dial to the agent and the pipe between them.
`serving` is a tls_store.ServingContext (or anything with ``listener_context()``, ``served()`` and ``number``); `windows` is a callable that returns the
current ``e2e_tunnel.Windows`` (idle and age); `host` is the device's own name, the only SNI it answers; `on_good(ip)` is told when a client address just
completed an authenticated call (the tunnel sends it to the node as a HINT). `load_target` and `prove` are the agent's surface file and the identity
proof (surfaces.load_target, surfaces.prove_listener): tests replace them, nothing else does."""
def __init__(self, *, host: str, serving: Any, windows: Callable[[], Any], budgets: EndpointBudgets | None = None,
on_good: Callable[[str], None] | None = None, load_target: Callable[[], Any] | None = None, prove: Callable[..., Any] | None = None,
local_port: int | None = None, handshake_timeout: float | None = None, head_timeout: float | None = None,
clock: Callable[[], float] = time.monotonic):
self.host = host
self.serving = serving
self.windows = windows
self.budgets = budgets if budgets is not None else EndpointBudgets(clock=clock)
self.on_good = on_good
self._load_target = load_target or surfaces.load_target
self._prove = prove or surfaces.prove_listener
self._local_port = local_port
self.handshake_timeout = policy.value("terminator.tls_handshake_s") if handshake_timeout is None else handshake_timeout
self.head_timeout = policy.value("terminator.request_head_s") if head_timeout is None else head_timeout
self._clock = clock
self.stats = TerminatorStats()
self._context: ssl.SSLContext | None = None
self._served_number: int | None = None
self.last_error = ""
# -- TLS
def tls_ready(self) -> bool:
"""TLS 1.3 is what the terminator will not serve without (plan section 3): a Python whose OpenSSL has none must say so instead of serving less
(tls_policy decides; the status file shows ``degraded`` / ``tls13_required``)."""
return tls_policy.tls13_available()
def _listener_context(self) -> ssl.SSLContext | None:
"""The one context that answers every handshake and follows the certificate pointer by itself; None while there is no certificate yet."""
if self._context is not None:
return self._context
try:
context = self.serving.listener_context()
except Exception as exc: # noqa: BLE001 - no certificate (or a store that cannot be read) is "no certificate", never a crash
self.last_error = getattr(exc, "word", type(exc).__name__.lower())
return None
if context.minimum_version < ssl.TLSVersion.TLSv1_3:
self.last_error = "tls13_required"
return None
wanted = self.host
follow = context.sni_callback
def only_my_name(connection, server_name, original):
if not server_name or server_name.lower().rstrip(".") != wanted:
return ssl.ALERT_DESCRIPTION_UNRECOGNIZED_NAME # no SNI, or another name: the handshake ends with an alert
return follow(connection, server_name, original) if follow is not None else None
context.sni_callback = only_my_name
self._context = context
self.last_error = ""
return context
async def _handshake(self, reader, writer, first: bytes) -> TLSStream | None:
ready = self.tls_ready()
context = self._listener_context() if ready else None
if context is None:
if ready:
self.stats.no_certificate += 1
else:
self.last_error = "tls13_required" # never silent: the tunnel reports tls13=False and the runner turns it into `degraded`
self.stats.handshake_failures += 1
return None
tls = TLSStream(reader, writer, context, initial=first)
try:
await tls.handshake(self.handshake_timeout)
except (ssl.SSLError, OSError, ConnectionError, asyncio.TimeoutError, ValueError) as exc:
self.stats.handshake_failures += 1
log.debug("TLS handshake failed (%s)", type(exc).__name__)
return None
if tls.selected_alpn() not in (None, policy.value("terminator.tls_alpn")):
self.stats.handshake_failures += 1
return None
self.stats.tls_streams += 1
number = getattr(self.serving, "number", None)
if number is not None and number != self._served_number:
self._served_number = number
with contextlib.suppress(Exception):
await asyncio.to_thread(self.serving.served) # a certificate that completed a handshake lets the previous generation go
return tls
# -- the agent
async def _dial_agent(self) -> tuple[asyncio.StreamReader, asyncio.StreamWriter] | None:
"""The agent's E2E listener, proven. Never LOCAL: surfaces.json names the E2E port (and is read again now: a restart of the agent moved it), the pid must
be alive, and the listener must answer the challenge on this very connection before anything of ChatGPT's is written to it."""
try:
target = self._load_target()
except surfaces.SurfaceIdentityError:
self.stats.no_surface += 1
return None
if target.port == self._local_surface_port():
self.stats.no_surface += 1 # a surfaces.json that names the LOCAL port is not an E2E listener (the bootstrap bearer works there)
return None
try:
reader, writer = await asyncio.wait_for(asyncio.open_connection(target.host, target.port, limit=1 << 16), LOCAL_DIAL_SECONDS)
except (OSError, asyncio.TimeoutError):
self.stats.agent_unreachable += 1
return None
try:
await self._prove(reader, writer, host=target.host, port=target.port, secret=target.secret, pid=target.pid, timeout=LOCAL_DIAL_SECONDS)
except surfaces.SurfaceIdentityError:
self.stats.identity_failures += 1
await close_writer(writer)
return None
return reader, writer
def _local_surface_port(self) -> int:
if self._local_port is not None:
return self._local_port
import os
return int(os.environ.get("ALL_MCP_PORT", "4410"))
# -- one stream
async def serve(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter, *, ip: str, first: bytes,
on_good: Callable[[str], None] | None = None) -> None:
"""One stream the node attached: `first` is what has already been read (the ClientHello); `on_good` replaces the terminator's own when this stream came
through a particular node (its HINT goes to that node). Returns when it ends; never raises."""
self.stats.streams += 1
self.stats.active += 1
tls: TLSStream | None = None
agent_writer: asyncio.StreamWriter | None = None
try:
tls = await self._handshake(reader, writer, first)
if tls is None:
return
agent_writer = await self._exchange(tls, ip, on_good or self.on_good)
except asyncio.CancelledError:
raise
except Exception as exc: # noqa: BLE001 - a stream must end quietly
log.debug("stream ended: %s", type(exc).__name__)
finally:
self.stats.active -= 1
await close_writer(agent_writer)
if tls is not None:
await tls.close()
async def _exchange(self, tls: TLSStream, ip: str, on_good: Callable[[str], None] | None) -> asyncio.StreamWriter | None:
"""The pipe between the client (through `tls`) and the agent, until either ends or a window runs out. Returns the agent's writer, which the caller closes."""
windows = self.windows()
framer = RequestFramer()
started = self._clock()
head_since = started # the first request has head_timeout from the end of the handshake, a later one from its first byte
last = started # the last byte that moved in either direction
forwarded = 0 # request heads passed on
first_good = False
first_status_seen = False
agent_writer: asyncio.StreamWriter | None = None
agent_task: asyncio.Task | None = None
agent_done: asyncio.Future = asyncio.get_running_loop().create_future()
async def refuse(status: int, reason: str, text: str, extra=()) -> None:
with contextlib.suppress(Exception):
await tls.write(http_response(status, reason, text, extra=extra))
async def agent_to_client(agent_reader: asyncio.StreamReader) -> None:
nonlocal last, first_status_seen
while True:
data = await agent_reader.read(READ_BYTES)
if not data:
return
if not first_status_seen:
first_status_seen = True
if first_good and data[:9] == b"HTTP/1.1 " and data[9:10] == b"2":
# a stranger cannot make the agent answer 2xx to an authenticated call: this address is a client of this device
self.budgets.note_client(ip)
if on_good is not None:
with contextlib.suppress(Exception):
on_good(ip)
last = self._clock()
self.stats.bytes_down += len(data)
await tls.write(data)
async def client_to_agent() -> None:
nonlocal last, head_since, forwarded, first_good, agent_writer, agent_task
while True:
now = self._clock()
if now - started >= windows.stream_max_age:
raise _Over("age")
if now - last >= windows.stream_idle:
raise _Over("idle")
waits = [windows.stream_idle - (now - last), windows.stream_max_age - (now - started)]
if framer.awaiting_head and (framer.partial_head or not forwarded):
if now - head_since >= self.head_timeout:
raise _Over("head")
waits.append(self.head_timeout - (now - head_since))
try:
data = await asyncio.wait_for(tls.read(), min(waits))
except asyncio.TimeoutError:
continue # the deadlines are looked at again above: traffic the other way may have moved the idle clock
if not data:
return
if forwarded and framer.awaiting_head and not framer.partial_head:
head_since = self._clock()
last = self._clock()
self.stats.bytes_up += len(data)
try:
segments = framer.feed(data)
except BadRequest as bad:
self.stats.count(self.stats.bad_requests, bad.word)
if not forwarded:
await refuse(bad.status, "Bad Request" if bad.status == 400 else "Request Rejected", bad.word)
return
for segment in segments:
if segment.kind == "head":
head = segment.head
bearer = (head.header(b"authorization") or b"")[:7].lower() == b"bearer "
endpoint = self.budgets.classify(head.path, credential_seen=bearer and self.budgets.is_client(ip))
if endpoint is not None:
allowed, retry = self.budgets.allow(endpoint, ip)
if not allowed:
self.stats.budget_refused += 1
if not forwarded:
await refuse(429, "Too Many Requests", "slow down", extra=(("Retry-After", str(int(retry) + 1)),))
return # on a connection that is already carrying a request, closing is the only answer that cannot be woven into its response
if not forwarded:
dialed = await self._dial_agent()
if dialed is None:
await refuse(503, "Service Unavailable", "ALL is not answering here")
return
agent_reader, agent_writer = dialed
keepalive(agent_writer)
agent_task = asyncio.create_task(agent_to_client(agent_reader))
agent_task.add_done_callback(lambda task: agent_done.done() or agent_done.set_result(task))
first_good = head.path in AUTHENTICATED_PATHS and (head.path == "/token" or bearer)
forwarded += 1
agent_writer.write(segment.data)
await agent_writer.drain()
up = asyncio.create_task(client_to_agent())
try:
done, _ = await asyncio.wait({up, agent_done}, return_when=asyncio.FIRST_COMPLETED)
if up in done:
over = up.exception()
if isinstance(over, _Over):
{"age": self._count_age, "idle": self._count_idle, "head": self._count_head}[over.word]()
return agent_writer
if over is not None:
raise over
# the client finished its side (close_notify, or an end of the stream): say so to the agent and keep delivering what it is still answering, for a while
if agent_task is not None and not agent_task.done():
with contextlib.suppress(Exception):
if agent_writer.can_write_eof():
agent_writer.write_eof()
with contextlib.suppress(asyncio.TimeoutError, Exception):
await asyncio.wait_for(asyncio.shield(agent_task), policy.value("timing.half_close_s"))
return agent_writer
task = agent_done.result() # the agent ended its answer, or the connection: the stream is over
if not task.cancelled() and task.exception() is not None and not isinstance(task.exception(), (ConnectionError, OSError)):
raise task.exception()
return agent_writer
finally:
for task in (up, agent_task):
if task is not None and not task.done():
task.cancel()
await asyncio.gather(*(t for t in (up, agent_task) if t is not None), return_exceptions=True)
await close_writer(agent_writer) # also when the client's side ended with an exception: the caller never gets the writer then
def _count_age(self) -> None:
self.stats.age_closes += 1
def _count_idle(self) -> None:
self.stats.idle_closes += 1
def _count_head(self) -> None:
self.stats.head_timeouts += 1
class _Over(Exception):
"""A window ran out (the stream's age, its silence, or the time a request head had): the stream is closed. `word` says which."""
def __init__(self, word: str):
super().__init__(word)
self.word = word
files/src/all_mcp/e2e_tunnel.py 1005 lines · 52.9 KB
"""ALL 2.0, DEVICE side of the tunnel, protocol v2 (plan sections 2, 3 and 5; the contract, sections 2 and 3): ChatGPT speaks ordinary HTTPS to a name that
is this computer.
A tunnel node (a small Go program on a public IP) reads only the SNI of a client's ClientHello and copies bytes. For every client of this computer it says ``OPEN`` on
the control connection this module keeps to it; the device dials the node back (``ATTACH``) and the raw stream, the client's own TLS, is handed to the terminator
(e2e_terminator.py), which ends it here with the certificate whose key only this computer has.
What this module is, in the order a connection lives:
* ``E2ETunnel`` keeps ONE control connection per node of the list the control plane gave this computer at enroll (``NodeLink``). Each one dials the node on its port with
TLS 1.3, ALPN ``all-tunnel/2`` and the node's public-CA certificate (certifi's bundle, the node's name, and an optional set of SPKI pins), and says
``HELLO 2 <session>``; the session is a v2 session of audience ``all-tunnel`` that ``device_credentials`` mints (and re-mints, with the control plane's time, when this
computer's clock is off). The node answers ``CHALLENGE``; the device answers ``PROOF`` (a DPoP proof over the NODE's URL, so the proof is good nowhere else); the node
asks the control plane and answers ``OK``: the host this computer serves (it must be the one the control plane gave THIS computer at its own enroll: a node
cannot rename a device), the status (active, gated, paused), when the connection ends (``valid_until``) and the windows it offers (ping, dead peer, idle, age).
The device clamps every window to its own policy and never trusts an offer beyond it.
* Before ``valid_until`` minus ``timing.renew_before_valid_until_s`` it sends ``RENEW`` with a NEW session (and the node asks the control plane again: this is how an
ended trial becomes ``gated`` without waiting for an event). ``PING`` every ``timing.ping_interval_s``; a node that is silent for ``timing.dead_peer_s`` is
dead. ``DRAIN`` makes the device prefer the other node; ``STATEMENT`` (the control plane's signed access statement, forwarded) is verified, must not be older than
the last one taken, and is handed to ``device_access.ingest``. A refused connection is retried with full-jitter back-off from ``timing.backoff_min_s`` to
``timing.backoff_max_s``; a wake from sleep or a change of network drops the connection at once and reconnects without back-off
(``timing.reconnect_after_wake_s`` is the bound the tests hold it to).
* Streams: the OPEN passes ``Admission`` (e2e_terminator.py) before the device does anything for it, then ``ATTACH`` on a new connection to the same node, then the
terminator. A good source address (one that just completed an authenticated call) is told to the node with ``HINT``.
The tunnel never trusts the node with anything but availability: a node can cut or delay, and cannot rename the device, extend its own grant past what the control plane
says, make it dial anything but the agent's one proven port, or lift a paywall (only signed statements do). Nothing of a client (an address, a name) is ever logged.
Nothing here runs on its own: all_mcp.e2e_runner starts it in a supervised child process of the relay service, after the enroll and the wave flag said so.
"""
from __future__ import annotations
import asyncio
import base64
import contextlib
import hashlib
import ipaddress
import json
import logging
import random
import re
import socket
import ssl
import time
from dataclasses import dataclass, field
from typing import Any, Callable
import certifi
from cryptography import x509
from cryptography.hazmat.primitives import serialization
from . import e2e_policy as policy
from .e2e_terminator import Admission, EndpointBudgets, TLS_HANDSHAKE_BYTE, Terminator, close_writer, keepalive
log = logging.getLogger("all_mcp.e2e_tunnel")
# Constants of this module that the policy does not define: how long a dial and the wait for an OK may take, how long a connection must have lived to count as
# stable (a node that accepts and closes in a loop does not reset the back-off), the least time between two RENEWs, and how the clock and the network are watched.
DIAL_SECONDS = 10.0
OK_WAIT_SECONDS = 30.0
STABLE_SECONDS = 30.0
RENEW_MIN_GAP_SECONDS = 5.0
RENEW_RETRY_SECONDS = (5.0, 15.0)
CLOCK_WATCH_SECONDS = 1.0
NETWORK_WATCH_SECONDS = 2.0
WALL_JUMP_SECONDS = 3.0
STALL_WAKE_SECONDS = 15.0
RESYNC_SECONDS = 20.0 # the limit of one background read of the control plane's time
REPLACED_WAIT_SECONDS = (5.0, 30.0)
DRAIN_RECHECK_SECONDS = 1.0 # how often a node that said DRAIN is looked at again to see whether the other node still serves
STATEMENTS_IN_FLIGHT = 4
# What a node can say (CLOSE <reason>) that means the control plane's answer about this computer has changed: the child reads its enroll again at once instead of
# waiting for its next turn (the kill switch must not take half an hour to reach a computer that is connected).
ATTENTION_WORDS = frozenset({"killed", "revoked", "released"})
HINT_RESEND_DIVISOR = 3
TUNNEL_PATH = policy.value("protocol.proof_htu_path")
_STATUSES = frozenset(policy.value("enums.authorize_status"))
_NODE_ID = re.compile(r"[a-z0-9]{1,16}")
_SECONDS = re.compile(r"(?:0|[1-9][0-9]{0,5})")
_STATEMENT = re.compile(r"[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+")
# ------------------------------------------------------------------------------------------------------------------------ errors
class TunnelError(Exception):
"""A connection that has to end. `word` is a fixed word (never text from the network) that the status file and the heartbeat may carry."""
def __init__(self, word: str):
super().__init__(word)
self.word = word
class ProtocolError(TunnelError):
"""A line that is not in the grammar (or arrives where it should not)."""
class NodeRefused(TunnelError):
"""The node answered ERR <code> (or closed with CLOSE <reason>)."""
class Kicked(Exception):
"""The connection was dropped on purpose (a wake from sleep, a change of network): reconnect at once, no back-off."""
# ------------------------------------------------------------------------------------------------------------------------ the lines
def valid_ip(text: Any) -> bool:
"""An address as the contract writes it: IPv4 dotted quad without leading zeros, or IPv6 in RFC 4291 text form, lower case, no zone, no brackets."""
if not isinstance(text, str) or text != text.lower() or any(char in text for char in "%[] ") or not text.isascii():
return False
try:
ipaddress.ip_address(text)
except ValueError:
return False
return True
def _bounded_json(text: str) -> Any:
def pairs(items: list[tuple[str, Any]]) -> dict[str, Any]:
keys = [key for key, _ in items]
if len(set(keys)) != len(keys):
raise ValueError("duplicate key")
return dict(items)
def refuse(_: str) -> Any:
raise ValueError("not a plain number")
return json.loads(text, object_pairs_hook=pairs, parse_float=refuse, parse_constant=refuse)
def _epoch(value: Any) -> bool:
return type(value) is int and 1 <= value <= 4102444800
def _ok_body(text: str) -> dict[str, Any]:
try:
body = _bounded_json(text)
except (ValueError, RecursionError):
raise ProtocolError("bad_command") from None
windows = ("ping_interval_s", "dead_peer_s", "stream_idle_s", "stream_max_age_s", "renew_before_valid_until_s")
ok = (isinstance(body, dict) and set(body) == {"v", "node", "host", "status", "valid_until", "server_time", "policy"}
and type(body["v"]) is int and body["v"] == policy.value("protocol.version")
and isinstance(body["node"], str) and _NODE_ID.fullmatch(body["node"]) is not None
and isinstance(body["host"], str) and len(body["host"]) <= policy.value("identity.fqdn_max_length")
and policy.anchored("identity.fqdn_regex").fullmatch(body["host"]) is not None
and body["status"] in _STATUSES and _epoch(body["valid_until"]) and _epoch(body["server_time"])
and isinstance(body["policy"], dict) and set(body["policy"]) == set(windows)
and all(type(body["policy"][name]) is int and 1 <= body["policy"][name] <= 86400 for name in windows))
if not ok:
raise ProtocolError("bad_command")
return body
def parse_node_line(line: str) -> dict[str, Any]:
"""One line the node sent (the text before the LF), as {"verb": ..., <field>: ...}. Raises ProtocolError. The grammar is contracts/schemas/wire-v2.grammar.json
(the tunnel client's tests run every n2d vector of contracts/vectors/lines.json through this function): a line is ASCII 0x20-0x7e, one space between fields,
no empty field, and it has its own maximum."""
if not isinstance(line, str) or not line or any(not 0x20 <= ord(char) <= 0x7E for char in line):
raise ProtocolError("bad_command")
verb, space, rest = line.partition(" ")
if verb == "OPEN":
limit = policy.value("protocol.open_line_max_bytes")
elif verb == "STATEMENT":
limit = policy.value("protocol.statement_line_max_bytes")
else:
limit = policy.value("protocol.line_max_bytes")
if len(line) + 1 > limit:
raise ProtocolError("line_too_long")
if space and not rest:
raise ProtocolError("bad_command")
if verb == "OK":
if not space:
raise ProtocolError("bad_command")
return {"verb": verb, "body": _ok_body(rest)}
fields = rest.split(" ") if space else []
if any(not field_ for field_ in fields):
raise ProtocolError("bad_command")
out: dict[str, Any] = {"verb": verb}
if verb == "CHALLENGE" and len(fields) == 1 and policy.anchored("identity.nonce_regex").fullmatch(fields[0]):
out["nonce"] = fields[0]
elif verb == "ERR" and len(fields) == 1 and fields[0] in policy.value("protocol.error_codes"):
out["code"] = fields[0]
elif (verb == "OPEN" and len(fields) == 4 and policy.anchored("identity.stream_id_regex").fullmatch(fields[0])
and policy.anchored("identity.attach_secret_regex").fullmatch(fields[1]) and valid_ip(fields[2])
and fields[3] in policy.value("protocol.source_classes")):
out.update(stream_id=fields[0], secret=fields[1], peer_ip=fields[2], cls=fields[3])
elif verb == "PONG" and not fields:
pass
elif verb == "STATEMENT" and len(fields) == 1 and _STATEMENT.fullmatch(fields[0]):
out["statement"] = fields[0]
elif verb == "DRAIN" and len(fields) == 1 and _SECONDS.fullmatch(fields[0]):
out["quiet_s"] = int(fields[0])
elif verb == "CLOSE" and len(fields) <= 1 and (not fields or fields[0] in policy.value("protocol.close_reasons")):
out["reason"] = fields[0] if fields else ""
else:
raise ProtocolError("bad_command")
return out
def _token_line(verb: str, token: str) -> str:
if (not isinstance(token, str) or not token.isascii() or len(token) > policy.value("protocol.jwt_max_bytes")
or _STATEMENT.fullmatch(token) is None):
raise ProtocolError("bad_command")
return f"{verb} {token}\n" if verb != "HELLO" else f"HELLO {policy.value('protocol.version')} {token}\n"
def hello_line(session: str) -> str:
return _token_line("HELLO", session)
def renew_line(session: str) -> str:
return _token_line("RENEW", session)
def proof_line(proof: str) -> str:
return _token_line("PROOF", proof)
def attach_line(stream_id: str, secret: str) -> str:
if (policy.anchored("identity.stream_id_regex").fullmatch(stream_id) is None
or policy.anchored("identity.attach_secret_regex").fullmatch(secret) is None):
raise ProtocolError("bad_command")
return f"ATTACH {stream_id} {secret}\n"
def hint_line(addresses: list[str]) -> str:
if not 1 <= len(addresses) <= policy.value("protocol.hint_max_addresses") or not all(valid_ip(address) for address in addresses):
raise ProtocolError("bad_command")
return "HINT " + " ".join(addresses) + "\n"
def close_line(reason: str = "") -> str:
if reason and reason not in policy.value("protocol.close_reasons"):
raise ProtocolError("bad_command")
return f"CLOSE {reason}\n" if reason else "CLOSE\n"
# ------------------------------------------------------------------------------------------------------------------------ windows
@dataclass(frozen=True)
class Windows:
"""The windows a connection works with: this device's own policy, or what a node offered clamped to it (never above)."""
ping_interval: float
dead_peer: float
stream_idle: float
stream_max_age: float
renew_before: float
@classmethod
def own(cls) -> "Windows":
return cls(float(policy.value("timing.ping_interval_s")), float(policy.value("timing.dead_peer_s")), float(policy.value("timing.stream_idle_s")),
float(policy.value("timing.stream_max_age_s")), float(policy.value("timing.renew_before_valid_until_s")))
def clamped(self, offered: dict[str, int]) -> "Windows":
"""What the node offered in its OK, held to this device's own windows: each is the smaller of the two (and at least one second, or this device's own window
when that is shorter: a window of nothing would close every stream at once). A dead-peer window that is not longer than the ping interval would drop a healthy
connection between two pings, so it is raised to twice the ping interval, still never above this device's own."""
def held(name: str, own: float) -> float:
return max(min(1.0, own), min(float(offered[name]), own))
ping = held("ping_interval_s", self.ping_interval)
dead = held("dead_peer_s", self.dead_peer)
if dead <= ping:
dead = min(self.dead_peer, ping * 2)
ping = min(ping, dead / 2)
return Windows(ping, dead, held("stream_idle_s", self.stream_idle), held("stream_max_age_s", self.stream_max_age),
held("renew_before_valid_until_s", self.renew_before))
# ------------------------------------------------------------------------------------------------------------------------ nodes and their certificates
@dataclass(frozen=True)
class NodeSpec:
"""A node of the list the control plane gave this computer: its id, name, port and optional SPKI pins (base64url SHA-256 of the SubjectPublicKeyInfo).
`address` (tests and a node reached by a fixed address) is where to connect; the name is still the one the certificate must carry."""
id: str
host: str
port: int
pins: frozenset[str] = frozenset()
address: str | None = None
@property
def url(self) -> str:
"""The node's own URL: what the DPoP proof is bound to (``https://<host>[:<port> unless 443]/all-tunnel/2``)."""
return f"https://{self.host}{'' if self.port == 443 else ':' + str(self.port)}{TUNNEL_PATH}"
@classmethod
def from_json(cls, value: dict[str, Any]) -> "NodeSpec":
return cls(value["id"], value["host"], int(value["port"]), frozenset(value.get("pins") or ()))
def spki_pin(der: bytes) -> str:
"""The pin of a certificate: base64url (no padding) of the SHA-256 of its SubjectPublicKeyInfo, so a renewal with the same key keeps the pin."""
public = x509.load_der_x509_certificate(der).public_key()
spki = public.public_bytes(serialization.Encoding.DER, serialization.PublicFormat.SubjectPublicKeyInfo)
return base64.urlsafe_b64encode(hashlib.sha256(spki).digest()).rstrip(b"=").decode("ascii")
def node_client_context() -> ssl.SSLContext:
"""TLS to a node: its certificate must chain to a public root (certifi's bundle: fresh or lagging machines see the same roots as the rest of ALL) and carry the
node's name, TLS 1.3 at least, and the ALPN that says this is the tunnel."""
context = ssl.create_default_context(cafile=certifi.where())
context.minimum_version = ssl.TLSVersion.TLSv1_3
context.set_alpn_protocols([policy.value("protocol.alpn")])
return context
# ------------------------------------------------------------------------------------------------------------------------ the network, as far as a change can be seen
# Interfaces a Mac raises and drops on its own (AirDrop's awdl/llw, VPN and iCloud Private Relay utun, the anpi/ap/bridge/gif/stf helpers): none of them is a move between
# networks, and counting them reconnected every tunnel each time AirDrop woke up.
VIRTUAL_INTERFACE_PREFIXES = ("lo", "loopback", "awdl", "llw", "utun", "anpi", "ap", "bridge", "gif", "stf")
def default_network_snapshot() -> tuple:
"""What changes when the computer moves between networks: the address the default routes would send from (a UDP connect sends nothing) and the interfaces that are
up. Cheap enough to take every couple of seconds."""
routes = []
for family, probe in ((socket.AF_INET, "192.0.2.1"), (socket.AF_INET6, "2001:db8::1")):
try:
with socket.socket(family, socket.SOCK_DGRAM) as probe_socket:
probe_socket.connect((probe, 9))
routes.append(probe_socket.getsockname()[0])
except OSError:
routes.append("")
try:
import psutil
up = tuple(sorted(name for name, stats in psutil.net_if_stats().items() if stats.isup and not name.lower().startswith(VIRTUAL_INTERFACE_PREFIXES)))
except Exception: # noqa: BLE001 - the routes alone are a good enough signal
up = ()
return (*routes, up)
# ------------------------------------------------------------------------------------------------------------------------ one control connection per node
@dataclass
class _Conn:
"""What one control connection is doing right now."""
reader: asyncio.StreamReader
writer: asyncio.StreamWriter
windows: Windows
exchange: str = "hello" # which exchange is under way: "hello" (first) or "renew"
awaiting: str | None = "challenge" # what the node owes: "challenge", then "ok"; None between exchanges
exchange_deadline: float = 0.0
session: str = "" # the session of the exchange under way
ready: bool = False
heard: float = 0.0
ping_at: float = 0.0
renew_at: float | None = None
valid_server: float = 0.0 # valid_until, on the control plane's clock
fetch: asyncio.Future | None = None
ready_at: float = 0.0
close_reason: str = ""
grace_ok: bool = False # the last RENEW was answered "control_unavailable": the control plane, not the node, is what does not answer
class NodeLink:
"""The control connection to one node, kept for as long as the tunnel runs: connect, authenticate, serve (OPEN, PING, RENEW, STATEMENT, DRAIN), and connect again
when it ends."""
def __init__(self, tunnel: "E2ETunnel", spec: NodeSpec):
self.tunnel = tunnel
self.spec = spec
self.state = "connecting" # connecting | ready | draining | down
self.status = "" # what the node last said about this computer: active | gated | paused
self.valid_until = 0 # the control plane's moment, from the last OK
self.last_error = ""
self.connects = 0
self.failures = 0
self.renewals = 0
self.drain_until = 0.0 # monotonic
self._writer: asyncio.StreamWriter | None = None
self._kick = asyncio.Event()
self._streams: set[asyncio.Task] = set()
self._task: asyncio.Task | None = None
self._hinted: dict[str, float] = {}
self._stopping = False
# -- life
def start(self) -> None:
if self._task is None:
self._task = asyncio.create_task(self.run(), name=f"e2e-node-{self.spec.id}")
async def stop(self) -> None:
self._stopping = True
self._kick.set()
if self._writer is not None:
with contextlib.suppress(Exception):
self._writer.write(close_line("shutdown").encode("ascii"))
if self._task is not None:
self._task.cancel()
await asyncio.gather(self._task, return_exceptions=True)
for task in list(self._streams):
task.cancel()
await asyncio.gather(*self._streams, return_exceptions=True)
def kick(self) -> None:
"""Drop the connection now and reconnect at once (a wake from sleep or a change of network: the connection is probably dead and only a timer would say so)."""
self._kick.set()
@property
def ready(self) -> bool:
return self.state in ("ready", "draining")
# -- the reconnect loop
def _delay(self, attempt: int) -> float:
"""Full-jitter back-off: somewhere between the policy's minimum and a ceiling that doubles with every failure up to its maximum."""
low, high = self.tunnel.backoff
return random.uniform(low, max(low, min(high, low * 2 ** (attempt + 1))))
async def run(self) -> None:
attempt = 0
while not self._stopping:
began = time.monotonic()
delay: float | None = None
try:
await self._connection()
except asyncio.CancelledError:
raise
except Kicked:
self.last_error = "network_changed"
delay, attempt = random.uniform(0.0, 1.0), 0
except NodeRefused as refused:
self.last_error = refused.word
delay = self._refusal_delay(refused.word, attempt)
if refused.word in ATTENTION_WORDS:
self.tunnel.attention.set()
if refused.word == "clock_skew":
with contextlib.suppress(Exception):
await asyncio.to_thread(self.tunnel.credentials.sync_clock, True) # the difference with the control plane is learned again
except TunnelError as error:
self.last_error = error.word
except ssl.SSLCertVerificationError:
self.last_error = "certificate_refused" # the node's certificate is not for its name, has expired or has no public root
except (ssl.SSLError, ConnectionError, OSError, asyncio.TimeoutError, asyncio.IncompleteReadError, asyncio.LimitOverrunError) as error:
self.last_error = "timeout" if isinstance(error, asyncio.TimeoutError) else ("tls_error" if isinstance(error, ssl.SSLError) else "connect_failed")
except Exception as error: # noqa: BLE001 - what a credential or a storage problem says is a closed word, never its text
self.last_error = self.tunnel.credential_word(error)
delay = self._refusal_delay(self.last_error, attempt) # a revoked computer or a license that is over is not asked again every second
if self.last_error == "internal_error":
log.warning("tunnel node %s: %s", self.spec.id, type(error).__name__)
finally:
was_ready = self.state in ("ready", "draining")
self.state = "down"
self._writer = None
self.tunnel.link_ended(self)
if self._stopping:
return
self.failures += 1
stable = was_ready and time.monotonic() - began >= STABLE_SECONDS
if stable:
attempt = 0
if delay is None:
delay = self._delay(attempt)
attempt += 1
self.state = "connecting"
await self._wait_to_reconnect(delay)
self._kick.clear()
async def _wait_to_reconnect(self, delay: float) -> None:
"""Wait `delay` before dialing again. A node that said DRAIN is left alone for its quiet period while the other node serves, and that is looked at again every
DRAIN_RECHECK_SECONDS: when the other node goes down too, this one is dialed at once rather than leaving the computer with no connection. A kick (wake, network
change) ends the wait at any time."""
until = time.monotonic() + delay
while True:
now = time.monotonic()
if self.drain_until > now and self.tunnel.other_node_ready(self):
left = max(until, self.drain_until) - now # the node said it is going down: the other node serves while it does
else:
left = until - now
if left <= 0:
return
try:
await asyncio.wait_for(self._kick.wait(), min(left, DRAIN_RECHECK_SECONDS))
return
except asyncio.TimeoutError:
continue
def _refusal_delay(self, word: str, attempt: int) -> float | None:
"""After a refusal: the plain back-off for a hiccup, the ceiling for an answer that will not change by itself (a revoked computer, a license that is over, a
node that names this computer something else) and a short wait for a node that closed because another connection of this computer replaced it."""
low, high = self.tunnel.backoff
if word in ("device_revoked", "license_inactive", "revoked", "released", "killed", "host_mismatch"):
return random.uniform(high / 2, high)
if word == "replaced":
return random.uniform(*REPLACED_WAIT_SECONDS)
return None
# -- dialing
async def dial(self, *, limit: int = 1 << 16) -> tuple[asyncio.StreamReader, asyncio.StreamWriter]:
"""TLS to the node, verified before a single byte is sent: its certificate chains to a public root and carries its name (the context), is on the pin list when
there is one, and the connection is TLS 1.3 with the ALPN of the tunnel."""
spec = self.spec
reader, writer = await asyncio.wait_for(
asyncio.open_connection(spec.address or spec.host, spec.port, ssl=self.tunnel.client_context(), server_hostname=spec.host,
ssl_handshake_timeout=DIAL_SECONDS, limit=limit), DIAL_SECONDS + 5)
try:
ssl_object = writer.get_extra_info("ssl_object")
if ssl_object is None or ssl_object.version() != "TLSv1.3" or ssl_object.selected_alpn_protocol() != policy.value("protocol.alpn"):
raise TunnelError("tls_error")
if spec.pins:
der = ssl_object.getpeercert(binary_form=True)
if not der or spki_pin(der) not in spec.pins:
raise TunnelError("pin_mismatch") # nothing was sent: the session never leaves this computer for a certificate that is not the pinned one
except BaseException:
await close_writer(writer)
raise
keepalive(writer)
return reader, writer
# -- one connection
async def _write(self, conn: _Conn, line: str) -> None:
conn.writer.write(line.encode("ascii"))
await asyncio.wait_for(conn.writer.drain(), conn.windows.dead_peer)
async def _connection(self) -> None:
tunnel = self.tunnel
reader, writer = await self.dial(limit=policy.value("protocol.statement_line_max_bytes") + 16)
self._writer = writer
conn = _Conn(reader, writer, tunnel.windows_own, heard=time.monotonic())
try:
conn.session = await tunnel.session(force=False)
await self._write(conn, hello_line(conn.session))
conn.exchange_deadline = time.monotonic() + OK_WAIT_SECONDS
await self._serve(conn)
finally:
if conn.fetch is not None and not conn.fetch.done():
conn.fetch.cancel()
await close_writer(writer)
def _timers(self, conn: _Conn, now: float) -> float:
"""Seconds until the next thing this connection has to do on its own."""
times = [conn.heard + conn.windows.dead_peer]
if conn.awaiting:
times.append(conn.exchange_deadline)
if conn.ready:
times.append(conn.ping_at)
if not conn.awaiting: # while a RENEW is under way the node decides (and the exchange has its own deadline)
times.append(self._local_valid_until(conn) + (self._grace_seconds() if conn.grace_ok else 0.0))
if conn.renew_at is not None and conn.fetch is None:
times.append(conn.renew_at)
return max(0.0, min(times) - now)
@staticmethod
def _grace_seconds() -> float:
"""How long past valid_until a connection may live while the control plane does not answer: the node's own grace for a paid computer (policy authorization.grace_paid_s)."""
return float(policy.value("authorization.grace_paid_s"))
def _local_valid_until(self, conn: _Conn) -> float:
"""valid_until as a reading of the monotonic clock: the control plane's moment minus its time now, plus now."""
return time.monotonic() + (conn.valid_server - self.tunnel.now())
async def _serve(self, conn: _Conn) -> None:
read_task: asyncio.Task | None = None
kick_task: asyncio.Task | None = None
try:
while True:
if read_task is None:
read_task = asyncio.ensure_future(conn.reader.readuntil(b"\n"))
if kick_task is None or kick_task.done():
kick_task = asyncio.ensure_future(self._kick.wait())
waiting = {read_task, kick_task} | ({conn.fetch} if conn.fetch is not None else set())
now = time.monotonic()
done, _ = await asyncio.wait(waiting, timeout=self._timers(conn, now), return_when=asyncio.FIRST_COMPLETED)
now = time.monotonic()
if kick_task in done:
self._kick.clear()
raise Kicked
if self._stopping:
return
if conn.fetch is not None and conn.fetch in done:
fetch, conn.fetch = conn.fetch, None
await self._start_renew(conn, fetch)
if read_task in done:
task, read_task = read_task, None
try:
raw = task.result()
except asyncio.IncompleteReadError:
raise TunnelError("closed_by_node") from None
except asyncio.LimitOverrunError:
raise ProtocolError("line_too_long") from None
conn.heard = now
await self._line(conn, raw[:-1].decode("ascii", "replace"), now)
await self._tick(conn, now)
finally:
for task in (read_task, kick_task):
if task is not None and not task.done():
task.cancel()
await asyncio.gather(*(t for t in (read_task, kick_task) if t is not None), return_exceptions=True)
async def _tick(self, conn: _Conn, now: float) -> None:
if now - conn.heard >= conn.windows.dead_peer:
raise TunnelError("dead_peer")
if conn.awaiting and now >= conn.exchange_deadline:
if not conn.ready:
raise TunnelError("handshake_timeout")
self._renew_failed(conn, now) # a RENEW the node never answered: the connection is still good until valid_until
if not conn.ready:
return
over = now - self._local_valid_until(conn)
if over >= 0 and not conn.awaiting and not (conn.grace_ok and over < self._grace_seconds()):
# The grant is over. A computer whose last RENEW got "control_unavailable" keeps its connection for the paid grace (the node decides who gets it: a trial or gated
# computer is closed by the node at valid_until whatever this side does, and a refusal of the control plane is never "the control plane is down"): closing here, at
# valid_until, made the grace useless: the device redialled, the node needed a good authorize for the new HELLO and the control plane was down.
raise TunnelError("expired")
if now >= conn.ping_at:
await self._write(conn, "PING\n")
conn.ping_at = now + conn.windows.ping_interval
if conn.renew_at is not None and now >= conn.renew_at and conn.fetch is None and not conn.awaiting:
conn.renew_at = None
conn.fetch = asyncio.ensure_future(self.tunnel.session(force=True))
async def _start_renew(self, conn: _Conn, fetch: asyncio.Future) -> None:
try:
session = fetch.result()
except asyncio.CancelledError:
raise
except Exception as error: # noqa: BLE001
self.last_error = self.tunnel.credential_word(error)
conn.grace_ok = self.last_error == "control_unreachable" # the control plane did not answer this computer's own request for a session: that is the outage the grace is for (a refusal is not)
self._renew_failed(conn, time.monotonic())
return
conn.session, conn.exchange, conn.awaiting = session, "renew", "challenge"
conn.exchange_deadline = time.monotonic() + OK_WAIT_SECONDS
await self._write(conn, renew_line(session))
def _renew_failed(self, conn: _Conn, now: float) -> None:
"""A renewal that did not come through: the old valid_until still holds, so try again soon, and well before it."""
conn.awaiting = None
left = self._local_valid_until(conn) - now
retry = random.uniform(*RENEW_RETRY_SECONDS)
conn.renew_at = now + (min(retry, max(0.5, left / 2)) if left > 0 else retry) # past valid_until (the grace) the retry keeps its normal pace, never a storm
async def _line(self, conn: _Conn, line: str, now: float) -> None:
message = parse_node_line(line)
verb = message["verb"]
if verb == "CHALLENGE":
if conn.awaiting != "challenge":
raise ProtocolError("unexpected_challenge")
proof = await self.tunnel.proof(conn.session, message["nonce"], self.spec.url)
conn.awaiting = "ok"
conn.exchange_deadline = now + OK_WAIT_SECONDS
await self._write(conn, proof_line(proof))
elif verb == "OK":
if conn.awaiting != "ok":
raise ProtocolError("unexpected_ok")
self._on_ok(conn, message["body"], now)
elif verb == "ERR":
if conn.ready:
self.last_error = message["code"]
if message["code"] == "control_unavailable":
conn.grace_ok = True # "the control plane did not answer" is what earns the paid grace
elif message["code"] != "busy":
conn.grace_ok = False # any other word is an answer of the control plane (a refusal): no grace. "busy" (the node's own pacing) says nothing about the control plane
self._renew_failed(conn, now) # a refused RENEW keeps the connection and its old valid_until
else:
raise NodeRefused(message["code"])
elif verb == "OPEN":
if conn.ready:
self.tunnel.on_open(self, message)
elif verb == "PONG":
return
elif verb == "STATEMENT":
self.tunnel.on_statement(message["statement"])
elif verb == "DRAIN":
self.drain_until = now + min(message["quiet_s"], policy.value("timing.drain_quiet_s")) # the grammar allows far more than the policy
if self.state == "ready":
self.state = "draining"
elif verb == "CLOSE":
raise NodeRefused(message["reason"] or "closed_by_node")
else:
raise ProtocolError("bad_command") # a verb the device only SENDS (HELLO, PING...) coming from the node
def _on_ok(self, conn: _Conn, body: dict[str, Any], now: float) -> None:
if body["host"] != self.tunnel.host:
raise TunnelError("host_mismatch") # the node says this computer is somebody else's name: it is not believed, nothing is served for it
server_now = self.tunnel.now()
limit = server_now + policy.value("authorization.valid_until_paid_s") + policy.value("authorization.node_skew_max_s")
valid = min(float(body["valid_until"]), limit)
if valid <= server_now:
raise TunnelError("grant_expired")
conn.windows = self.tunnel.windows_own.clamped(body["policy"])
conn.valid_server = valid
conn.grace_ok = False
conn.awaiting = None
self.status, self.valid_until = body["status"], int(valid)
if conn.exchange == "renew":
self.renewals += 1
if not conn.ready:
conn.ready, conn.ready_at = True, now
# A draining node refuses a new HELLO and sends DRAIN right after the OK of one that was already authenticating (tunnel/server.go followUp), so
# an OK on a new connection is a node that is not draining now, whatever the previous connection was told: a DRAIN that follows sets it again.
self.drain_until = 0.0
self.state = "ready"
self.connects += 1
self.last_error = ""
conn.ping_at = now + conn.windows.ping_interval
self.tunnel.link_ready(self)
self.tunnel.windows_changed(conn.windows)
wait = valid - server_now - conn.windows.renew_before
conn.renew_at = now + max(wait, min(RENEW_MIN_GAP_SECONDS, (valid - server_now) / 2)) + random.uniform(0, min(10.0, max(0.0, wait) * 0.05))
# -- what streams and statements ask of the control connection
def send_hint(self, addresses: list[str]) -> None:
"""Tell the node about client addresses that just completed authenticated calls (each at most once in a third of the node's memory of them)."""
writer = self._writer
if writer is None or self.state not in ("ready", "draining"):
return
now = time.monotonic()
fresh = [address for address in addresses if now - self._hinted.get(address, float("-inf")) >= policy.value("node_limits.hint_ttl_s") / HINT_RESEND_DIVISOR]
for start in range(0, len(fresh), policy.value("protocol.hint_max_addresses")):
batch = fresh[start:start + policy.value("protocol.hint_max_addresses")]
try:
writer.write(hint_line(batch).encode("ascii"))
except (ProtocolError, OSError):
return
for address in batch:
self._hinted[address] = now
while len(self._hinted) > policy.value("surface.limiter_lru_max"):
self._hinted.pop(next(iter(self._hinted)))
def track(self, task: asyncio.Task) -> None:
self._streams.add(task)
task.add_done_callback(self._streams.discard)
def snapshot(self) -> dict[str, Any]:
return {"id": self.spec.id, "state": self.state, "status": self.status, "valid_until": self.valid_until or None, "last_error": self.last_error,
"connects": self.connects, "failures": self.failures, "renewals": self.renewals, "streams": len(self._streams)}
# ------------------------------------------------------------------------------------------------------------------------ the tunnel
def _statement_seq(token: str) -> int | None:
"""The `seq` of a statement that was already verified (the signature is what makes it believable; this only reads it)."""
try:
payload = token.split(".")[1]
claims = json.loads(base64.urlsafe_b64decode(payload + "=" * (-len(payload) % 4)))
seq = claims.get("seq")
return seq if type(seq) is int and seq >= 1 else None
except (IndexError, ValueError, UnicodeError, AttributeError):
return None
@dataclass
class TunnelStats:
streams_opened: int = 0
streams_refused: dict[str, int] = field(default_factory=dict)
attach_failures: int = 0
statements_taken: int = 0
statements_refused: int = 0
wakes: int = 0
network_changes: int = 0
def as_dict(self) -> dict[str, Any]:
return dict(self.__dict__)
class E2ETunnel:
"""The device's side of the tunnel: a control connection to every node, the terminator for every stream, and the watchers of the clock and the network.
`host` is the name this computer serves (from its own enroll); `nodes` the list the control plane gave it; `credentials` is a device_credentials.DeviceCredentials (or
anything with ``tunnel_session``, ``tunnel_proof``, ``now``, ``sync_clock``, ``verify_access`` and ``clock``); `serving` a tls_store.ServingContext. `ingest` takes a
verified statement (device_access.ingest). Everything else exists so that tests can drive it without a network, a clock or an agent."""
def __init__(self, *, host: str, nodes: list[NodeSpec], credentials: Any, serving: Any, ingest: Callable[[str], bool] | None = None,
client_context: Callable[[], ssl.SSLContext] = node_client_context, own_windows: Windows | None = None,
backoff: tuple[float, float] | None = None, admission: Admission | None = None, budgets: EndpointBudgets | None = None,
terminator: Terminator | None = None, terminator_options: dict[str, Any] | None = None,
network_snapshot: Callable[[], Any] = default_network_snapshot, wall: Callable[[], float] = time.time,
monotonic: Callable[[], float] = time.monotonic):
self.host = host
self.credentials = credentials
self._client_context = client_context
self.windows_own = own_windows or Windows.own()
self._windows = self.windows_own
self.backoff = backoff or (float(policy.value("timing.backoff_min_s")), float(policy.value("timing.backoff_max_s")))
self.admission = admission or Admission()
self.terminator = terminator or Terminator(host=host, serving=serving, windows=lambda: self._windows,
budgets=budgets, **(terminator_options or {}))
if ingest is None:
from . import device_access
ingest = device_access.ingest
self._ingest = ingest
self._network_snapshot = network_snapshot
self._wall = wall
self._monotonic = monotonic
self.stats = TunnelStats()
self.links: dict[str, NodeLink] = {}
self._last_seq = 0
self._statements = 0
self._stop = asyncio.Event()
self._watch_task: asyncio.Task | None = None
self._resync_task: asyncio.Task | None = None
self._changed = asyncio.Event()
self._client_ctx: ssl.SSLContext | None = None
self.attention = asyncio.Event() # a node said something that changes what the control plane thinks of this computer: read the enroll again now
self.set_nodes(nodes)
# -- the nodes
def set_nodes(self, nodes: list[NodeSpec]) -> None:
"""The list the control plane gave: a node that is new gets a connection, one that is gone loses it, one whose name, port or pins changed gets a fresh one."""
wanted = {spec.id: spec for spec in nodes}
for node_id, link in list(self.links.items()):
if node_id not in wanted or link.spec != wanted[node_id]:
del self.links[node_id]
if self._watch_task is not None: # running: end it in the background
asyncio.get_running_loop().create_task(link.stop())
for node_id, spec in wanted.items():
if node_id not in self.links:
self.links[node_id] = NodeLink(self, spec)
if self._watch_task is not None:
self.links[node_id].start()
def other_node_ready(self, link: NodeLink) -> bool:
return any(other is not link and other.state == "ready" for other in self.links.values())
# -- what the links ask for (blocking credential work runs in a thread)
def client_context(self) -> ssl.SSLContext:
"""The TLS context for every connection to a node, built once (loading the root bundle for each stream would cost more than the stream)."""
if self._client_ctx is None:
self._client_ctx = self._client_context()
return self._client_ctx
def now(self) -> float:
"""The control plane's time (the offset the credentials learned): what valid_until is measured against."""
return self.credentials.now()
async def session(self, *, force: bool) -> str:
return await asyncio.to_thread(self.credentials.tunnel_session, force)
async def proof(self, session: str, nonce: str, node_url: str) -> str:
return await asyncio.to_thread(self.credentials.tunnel_proof, session, nonce, node_url)
def credential_word(self, error: BaseException) -> str:
"""The closed word for a credential failure (never its text)."""
code = getattr(error, "code", "")
if code in ("device_revoked", "license_inactive", "device_identity_required", "device_owned_by_other_license", "invalid_license"):
return {"device_owned_by_other_license": "device_revoked", "invalid_license": "license_inactive"}.get(code, code)
if getattr(error, "retryable", False):
return "control_unreachable"
return "internal_error" if not hasattr(error, "code") else "session_refused"
# -- links report
def link_ready(self, link: NodeLink) -> None:
self._changed.set()
def link_ended(self, link: NodeLink) -> None:
self._changed.set()
def windows_changed(self, windows: Windows) -> None:
self._windows = windows # the windows the newest OK clamped: streams that start from now on use them
# -- streams
def on_open(self, link: NodeLink, message: dict[str, Any]) -> None:
if link.status == "paused":
return # a node does not open streams for a paused computer; one that does is not served
ticket, why = self.admission.admit(message["peer_ip"], message["cls"])
if ticket is None:
refused = self.stats.streams_refused
refused[why] = refused.get(why, 0) + 1
return # no ATTACH: the node closes the client when the attach window ends
self.stats.streams_opened += 1
link.track(asyncio.create_task(self._stream(link, message, ticket)))
async def _stream(self, link: NodeLink, message: dict[str, Any], ticket) -> None:
writer: asyncio.StreamWriter | None = None
try:
try:
reader, writer = await link.dial()
writer.write(attach_line(message["stream_id"], message["secret"]).encode("ascii"))
await asyncio.wait_for(writer.drain(), policy.value("protocol.attach_timeout_s"))
first = await asyncio.wait_for(reader.read(1 << 16), self.terminator.handshake_timeout)
except asyncio.CancelledError:
raise
except Exception: # noqa: BLE001 - a stream that cannot be attached is just not served
self.stats.attach_failures += 1
return
if not first or first.startswith(b"ERR") or first[0] != TLS_HANDSHAKE_BYTE:
self.stats.attach_failures += 1 # the node refused the attach, or what came is not a TLS handshake: no client of ChatGPT starts with anything else
return
await self.terminator.serve(reader, writer, ip=message["peer_ip"], first=first, on_good=lambda ip, link=link: link.send_hint([ip]))
finally:
self.admission.release(ticket)
await close_writer(writer)
# -- statements
def on_statement(self, token: str) -> None:
if self._statements >= STATEMENTS_IN_FLIGHT:
return # a node that floods statements gets nothing more verified than it can be served
self._statements += 1
task = asyncio.create_task(self._take_statement(token))
task.add_done_callback(lambda _: setattr(self, "_statements", self._statements - 1))
async def _take_statement(self, token: str) -> None:
"""A signed statement the node forwarded: taken only when it verifies against the control plane's keys (a forgery and a statement for another computer do
not), carries a `seq`, and that `seq` is not lower than the highest taken so far (an old `active` replayed cannot lift a paywall). The same `seq` again is
harmless. device_access.ingest then decides whether it is newer than what the computer holds."""
try:
await asyncio.to_thread(self.credentials.verify_access, token)
except Exception: # noqa: BLE001 - not believed
self.stats.statements_refused += 1
return
seq = _statement_seq(token)
if seq is None or seq < self._last_seq:
self.stats.statements_refused += 1
return
self._last_seq = seq
try:
taken = await asyncio.to_thread(self._ingest, token)
except Exception: # noqa: BLE001
taken = False
if taken:
self.stats.statements_taken += 1
else:
self.stats.statements_refused += 1
# -- running
async def run(self) -> None:
"""Keep every node's control connection up until stop()."""
self._watch_task = asyncio.create_task(self._watch())
for link in self.links.values():
link.start()
try:
await self._stop.wait()
finally:
background = [task for task in (self._watch_task, self._resync_task) if task is not None]
for task in background:
task.cancel()
await asyncio.gather(*background, *(link.stop() for link in list(self.links.values())), return_exceptions=True)
def stop(self) -> None:
self._stop.set()
def kick_all(self, reason: str) -> None:
"""Drop every control connection now (wake, network change): each reconnects within a second or so, with no back-off."""
if reason == "wake":
self.stats.wakes += 1
else:
self.stats.network_changes += 1
for link in self.links.values():
link.kick()
def _resync_clock(self) -> None:
"""Learn the control plane's time again in the background, never ahead of the reconnection: one at a time, each with its own limit."""
if self._resync_task is not None and not self._resync_task.done():
return
async def resync() -> None:
with contextlib.suppress(Exception):
await asyncio.wait_for(asyncio.to_thread(self.credentials.sync_clock, True), RESYNC_SECONDS)
self._resync_task = asyncio.create_task(resync(), name="e2e-clock-resync")
async def _watch(self) -> None:
"""Notice a wake from sleep, a step of the clock and a change of network, which no timer can: a connection that was open when the lid closed is dead and says
nothing. The wall clock moving more than the monotonic one (macOS and Linux do not count sleep in the monotonic clock) or the loop not running for a long
while (Windows does count it) is a wake; the routes and the interfaces that are up changing is a network change. Either one drops the connections."""
last_wall, last_mono = self._wall(), self._monotonic()
last_net = await asyncio.to_thread(self._network_snapshot)
last_net_at = last_mono
while True:
await asyncio.sleep(CLOCK_WATCH_SECONDS)
wall, mono = self._wall(), self._monotonic()
jump = (wall - last_wall) - (mono - last_mono)
stalled = mono - last_mono > STALL_WAKE_SECONDS
last_wall, last_mono = wall, mono
if abs(jump) > WALL_JUMP_SECONDS or stalled:
# the connections start over at once (the network may not be usable yet, and a request waiting for it would keep the dead ones up); the difference
# between this computer's clock and the control plane's, learned before, is learned again alongside
self.kick_all("wake")
self._resync_clock()
last_net = await asyncio.to_thread(self._network_snapshot)
last_net_at = mono
continue
if mono - last_net_at >= NETWORK_WATCH_SECONDS:
last_net_at = mono
try:
current = await asyncio.to_thread(self._network_snapshot)
except Exception: # noqa: BLE001
continue
if current != last_net:
last_net = current
self.kick_all("network")
self._resync_clock()
# -- what the status file shows
def state(self) -> str:
"""connecting (no node yet), ready (every node), degraded (some, not all), clock_skew (no node, and the answer was that the clock is wrong)."""
links = list(self.links.values())
ready = [link for link in links if link.ready]
if links and len(ready) == len(links):
return "ready"
if ready:
return "degraded"
if any(link.last_error == "clock_skew" for link in links):
return "clock_skew"
return "connecting"
def snapshot(self) -> dict[str, Any]:
links = list(self.links.values())
return {"state": self.state(), "nodes": [link.snapshot() for link in links], "connected": sum(1 for link in links if link.ready), "configured": len(links),
"last_error": next((link.last_error for link in links if link.last_error), ""), "stats": self.stats.as_dict(),
"terminator": self.terminator.stats.as_dict(), "tls13": self.terminator.tls_ready(),
"windows": {"ping_interval_s": self._windows.ping_interval, "dead_peer_s": self._windows.dead_peer,
"stream_idle_s": self._windows.stream_idle, "stream_max_age_s": self._windows.stream_max_age}}
files/src/all_mcp/surfaces.py 865 lines · 46.5 KB
"""The agent's surfaces (ALL 2.0, end-to-end encryption): who may reach what is decided by WHICH SOCKET a request came in on, never by what it says.
One inner app (the MCP stack, OAuth, the pause and tray routes) is served by up to three uvicorn servers in one event loop:
* LOCAL 127.0.0.1:4410 (ALL_MCP_PORT). The ALL icon, the command line, the installer and the QA scripts (and the legacy relay's client while
the lane is off). Nothing about the request changes anything: the bootstrap bearer works, the routes of the icon exist, and the paywall gate is not
here (the old relay gates for the computers it still serves, because it can still read the request). It is the ONLY exempt surface.
* E2E 127.0.0.1:<ephemeral port>, written to state_dir()/surfaces.json (0600, with the agent's pid and a secret that is new at every start). Only
the computer's own TLS terminator dials it, with what ChatGPT sent. So everything that arrives here is remote, whatever it claims to be:
it is closed by default (http only, one Host, an exact list of paths, size caps, no forwarding headers, no bootstrap bearer), the trial's
paywall and the activation funnel always apply, and they fail closed (no answer from the control plane, or no device id: blocked).
* RELAY 127.0.0.1:<ephemeral port>, written to state_dir()/relay-lane.json (0600, same shape as surfaces.json). Only the legacy relay's client dials it (ALL 2.0
WP11; `ALL_MCP_RELAY_LANE`, on unless it says off). It carries what the old relay forwards, which is remote traffic that the relay terminated and can read:
so it gets the relay's own allowlist (the paths of its DEVICE_PATH), one Host, size caps, no forwarding headers and no bootstrap bearer, and nothing else
of LOCAL (no pause, tray, permissions or pairing-start route, no /public). The paywall gate is NOT here: the relay gates what it serves.
Every wrapper writes the surface into `scope["state"]` (the listener marks LOCAL, E2E and RELAY; no header can: `x-all-surface: local` is stripped on E2E and RELAY and
read nowhere). The authentication (middleware.AuthMiddleware) refuses the bootstrap bearer on E2E and RELAY like an unknown token, and the gate
(middleware.DeviceGateMiddleware) enforces on E2E and passes the others. A request that carries no mark (a test, a developer's `build_app()`) is LOCAL-like: only
the wrappers below put traffic in front of the app in production.
Why the identity proof: the terminator reads where to dial from surfaces.json, and a port that the agent freed (it crashed, it restarted) can be bound by
another process, which would then receive ChatGPT's tokens in clear. So, before the terminator forwards the first ChatGPT byte, it sends on that same
connection a reserved GET with an HMAC challenge built from the per-start secret; only the agent holds the secret and answers
(`prove_listener`, the terminator's half); anything else, a 404 or silence, is refused and nothing of ChatGPT's is sent.
The relay lane has the same identity proof (relay_client.RelayClient, the dialer, runs `prove_listener` against it before it trusts relay-lane.json).
The E2E listener is started lazily and supervised (SurfaceRunner): only when the rollback switch ALL_MCP_E2E is not off, ALL_MCP_REMOTE_SURFACE is not 0 and
runtime.env names an enrolled label; inside its own exception boundary, so a failure there (it cannot bind, it crashes) leaves the LOCAL surface serving; its
lifespan is off (the inner app's lifespan runs once, under LOCAL); and uvicorn runs on both with `proxy_headers` off, no trusted forwarders, a keep-alive and a
concurrency ceiling, and an h11 head cap (X-Forwarded-For is never believed: the client of a request is its socket's peer).
"""
from __future__ import annotations
import asyncio
import base64
import contextlib
import contextvars
import hashlib
import hmac
import json
import logging
import os
import re
import secrets
import socket
import stat
import time
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Callable
from . import e2e_policy as policy
from . import state_io
from .config import runtime_values, state_dir
_LOG = logging.getLogger(__name__)
LOOPBACK = "127.0.0.1"
SURFACES_FILE = "surfaces.json"
LANE_FILE = "relay-lane.json"
# What a wrapper writes into scope["state"], and the two values it can have.
SURFACE_KEY = "all_mcp_surface"
LOCAL = "local"
E2E = "e2e"
RELAY = "relay"
# Both surfaces that carry remote traffic: what the authentication must never treat as the computer's own tools.
REMOTE = frozenset({E2E, RELAY})
# The E2E wrapper also writes the name it was started for (the device's own, from runtime.env): the origin of an E2E request is the one of that name, so the issuer,
# the resource and the OAuth store follow the listener and never the Host header a caller wrote (origins.OriginSet.for_surface).
E2E_HOST_KEY = "all_mcp_surface_host"
# The rollback switch (one name; ALL_E2E is only an alias of the developer's dev runner) and the switch that keeps the E2E listener from
# starting at all (the update candidate probe sets it so that the probe is exactly the 1.1 probe).
SWITCH = "ALL_MCP_E2E"
REMOTE_SURFACE = "ALL_MCP_REMOTE_SURFACE"
# The hardened lane of the legacy relay (WP11). ON unless it says off; ALL_MCP_REMOTE_SURFACE=0 (the update candidate probe) also keeps it from starting.
RELAY_LANE = "ALL_MCP_RELAY_LANE"
# The old route's own socket (the relay client's WebSocket to the relay). ON whenever the variable is absent or empty: a 1.0 or 1.1 runtime.env that has no encrypted-path
# keys, updated to 2.0, stays connected and answering. Only an explicit off value stops it (a later 2.x, on the owner's order, or "Cerrar la ruta antigua" on one computer).
LEGACY_RELAY = "ALL_MCP_LEGACY_RELAY"
# The canary device's mode, "certificate manager only" (the tunnel runbook, section 8): the e2e child keeps this computer's certificate issued and renewed as usual but opens no tunnel
# connection of its own, because the canary device app holds the connection to the nodes under the same label (two clients of one label evict each other for ever). The opposite
# default of the rollback switches: it is ON only when it says on, so a computer that never set it is an ordinary one. CANARY_STATUS is the canary's own --status-file, where the child
# reads whether its links are up (e2e_canary.py). Both are kept by every rewrite of runtime.env (config.E2E_RUNTIME_KEYS) and neither is ever written by an install or an update.
CERT_ONLY = "ALL_E2E_CERT_ONLY"
CANARY_STATUS = "ALL_E2E_CANARY_STATUS"
_OFF = frozenset({"0", "off", "false", "no"})
_ON = frozenset({"1", "on", "true", "yes"})
# The runtime.env file, written by the installer when the computer enrolls: the label and the host the control plane gave it (and, optionally, the zone it must be in).
LABEL_KEY, HOST_KEY, ZONE_KEY = "ALL_E2E_LABEL", "ALL_E2E_HOST", "ALL_E2E_ZONE"
# What the E2E surface answers, by exact path. Everything else is a 404: the pause, resume, pairing start, tray,
# permissions and public routes are not here even when the app has them, and `/mcp/` is served as `/mcp` (no redirect, ChatGPT keeps the slash it was given).
E2E_PATHS = frozenset({
"/mcp", "/mcp/", "/register", "/authorize", "/token", "/pair", "/healthz",
"/.well-known/oauth-protected-resource", "/.well-known/oauth-protected-resource/mcp",
"/.well-known/oauth-authorization-server", "/.well-known/openid-configuration",
})
# What the lane answers: the paths of the relay's DEVICE_PATH (the relay index module), which is what the old relay forwards; the relay sends "/" + the path after
# /d/<device>, so a bare /d/<device> and /d/<device>/ arrive as "/". The lane is the E2E list plus "/": no pause, tray, permissions, pairing-start or /public path is on
# it, even though the app has them. The relay lane test suite reads the relay's regex and checks both lists against each other.
LANE_PATHS = E2E_PATHS | {"/"}
# Headers that say who the caller is or how it got here: a remote caller writes them as it likes, so none of them reaches the app.
_STRIPPED_PREFIXES = (b"x-forwarded-", b"cf-", b"x-all-")
_STRIPPED_NAMES = frozenset({b"forwarded"})
# Ceilings of the two uvicorn servers that are not in the policy table: LOCAL carries the legacy relay's one connection per request, so it is far more
# generous than E2E (which gets twice the terminator's stream cap: its streams plus the identity probes in flight).
LOCAL_CONCURRENCY = 512
E2E_CONCURRENCY_FACTOR = 2
GRACEFUL_SHUTDOWN_SECONDS = 10
START_TIMEOUT_SECONDS = 10.0
WATCH_SECONDS = 5.0 # how often the runner looks again at whether the E2E listener should be running
_current: contextvars.ContextVar[str | None] = contextvars.ContextVar("all_mcp_surface", default=None)
_serving = {"e2e": False}
def current() -> str | None:
"""The surface of the request being served in this task (E2E, LOCAL), or None outside a request: what the funnel asks, because it is called deep inside
the OAuth route and the gate, far from the ASGI scope."""
return _current.get()
def in_e2e() -> bool:
return _current.get() == E2E
def e2e_serving() -> bool:
"""True while this process has an E2E listener up: what keeps the paywall verdict fresh (device_access.start_refresher)."""
return _serving["e2e"]
def surface_of(scope: dict) -> str | None:
return (scope.get("state") or {}).get(SURFACE_KEY)
# ------------------------------------------------------------------------------------------------------------------------ is the E2E listener wanted?
def _setting(values: dict[str, str], name: str) -> str:
return (os.environ.get(name) or values.get(name, "")).strip().lower()
@dataclass(frozen=True)
class E2EConfig:
label: str
host: str # the device's full name, exactly what a caller must write in Host
def enrolled_host(values: dict[str, str] | None = None) -> E2EConfig | None:
"""The label and host this computer enrolled with, when runtime.env holds a well-formed pair (label as the policy writes it, host one label under a zone,
and under ALL_E2E_ZONE when that is set), else None. A computer without them has no encrypted name, so nothing listens for one."""
values = runtime_values() if values is None else values
label, host, zone = values.get(LABEL_KEY, ""), values.get(HOST_KEY, ""), values.get(ZONE_KEY, "")
if (not policy.anchored("identity.label_regex").fullmatch(label) or len(host) > policy.value("identity.fqdn_max_length")
or not policy.anchored("identity.fqdn_regex").fullmatch(host) or not host.startswith(label + ".")):
return None
if zone and not policy.host_pattern(zone).match(host):
return None
return E2EConfig(label, host)
def e2e_config(values: dict[str, str] | None = None) -> E2EConfig | None:
"""What the E2E listener should serve right now, or None when it should not run: the rollback switch is off, the remote surface is off, or the runtime has
no enrolled label. The ONE rollback variable of the migration is ALL_MCP_E2E: it stops the listener (and the tunnel, whose own code reads the same name);
it never makes the paywall easier, because a computer that has no E2E surface has nothing the paywall would guard."""
values = runtime_values() if values is None else values
return None if e2e_switched_off(values) else enrolled_host(values)
def e2e_switched_off(values: dict[str, str] | None = None) -> bool:
"""True while the rollback switch (ALL_MCP_E2E) or the remote surface (ALL_MCP_REMOTE_SURFACE) is off: no E2E listener runs, so nothing that hands out the encrypted address
(the reconnect flow, the tray's status) may either."""
values = runtime_values() if values is None else values
return _setting(values, SWITCH) in _OFF or _setting(values, REMOTE_SURFACE) in _OFF
def relay_lane_wanted(values: dict[str, str] | None = None) -> bool:
"""Whether the agent serves the relay lane and the relay's client dials it: ALL_MCP_RELAY_LANE is not off (it is ON when absent) and ALL_MCP_REMOTE_SURFACE is not
off. The agent and the legacy relay's client both ask THIS, over the same runtime.env, so they never disagree (a lane the agent does not serve is a lane the client
does not wait for). Off leaves the legacy relay on the LOCAL surface exactly as 1.1 had it."""
values = runtime_values() if values is None else values
return _setting(values, RELAY_LANE) not in _OFF and _setting(values, REMOTE_SURFACE) not in _OFF
def legacy_relay_wanted(values: dict[str, str] | None = None) -> bool:
"""Whether the relay client keeps the old route's WebSocket: ALL_MCP_LEGACY_RELAY is not an explicit off value. Absent, empty or anything else means ON, so that the
absence of the 2.0 keys can never switch a computer's only working route off."""
values = runtime_values() if values is None else values
return _setting(values, LEGACY_RELAY) not in _OFF
def cert_only(values: dict[str, str] | None = None) -> bool:
"""Whether this computer is a canary's: ALL_E2E_CERT_ONLY says on (1, on, true or yes; absent, empty or anything else is off). The e2e child then runs the enroll, the certificate
manager and the self-check of the public name exactly as always, and starts no tunnel (e2e_runner.ChildStages). The rollback switch still wins: ALL_MCP_E2E=off stops all of it."""
values = runtime_values() if values is None else values
return _setting(values, CERT_ONLY) in _ON
# ------------------------------------------------------------------------------------------------------------------------ the transport security of the inner app
_transport_settings: list[Any] = []
def adopt_transport_security(settings: Any, values: dict[str, str] | None = None) -> None:
"""The inner app's DNS-rebinding settings (mcp TransportSecuritySettings, built by mcp_server.build_app) learn the E2E host: the Host the E2E surface lets
through must also be one the MCP transport accepts. Called when the app is built, and again by the runner when the listener starts (a computer that
enrolls while the agent runs)."""
_transport_settings.append(settings)
found = enrolled_host(values)
if found is not None:
_allow_host(settings, found.host)
def _allow_host(settings: Any, host: str) -> None:
for field, entry in (("allowed_hosts", host), ("allowed_origins", f"https://{host}")):
listed = getattr(settings, field)
if entry not in listed:
listed.append(entry)
def _allow_everywhere(host: str) -> None:
for settings in list(_transport_settings):
_allow_host(settings, host)
# ------------------------------------------------------------------------------------------------------------------------ the identity proof (T14)
def b64u(raw: bytes) -> str:
return base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii")
def _key(secret: str) -> bytes:
raw = base64.urlsafe_b64decode(secret + "=" * (-len(secret) % 4))
if len(raw) != 32 or not policy.anchored("identity.attach_secret_regex").fullmatch(secret):
raise ValueError("surface secret")
return raw
def new_secret() -> str:
return b64u(secrets.token_bytes(32))
def challenge_mac(secret: str, nonce: str) -> str:
"""What the terminator puts on its challenge: only a holder of the per-start secret can answer it."""
return b64u(hmac.new(_key(secret), b"all-surface-challenge-v1\n" + nonce.encode("ascii"), hashlib.sha256).digest())
def surface_proof(secret: str, nonce: str, pid: int, port: int) -> str:
"""What the listener answers: it proves it holds the secret of the process surfaces.json names, for THIS nonce, pid and port."""
message = b"all-surface-proof-v1\n" + nonce.encode("ascii") + b"\n" + str(pid).encode() + b"\n" + str(port).encode()
return b64u(hmac.new(_key(secret), message, hashlib.sha256).digest())
def verify_challenge(secret: str, nonce: Any, mac: Any) -> bool:
"""The agent's check of a challenge: a nonce the policy allows, and the MAC only a holder of the secret can make (compared in constant time)."""
try:
return (isinstance(nonce, str) and policy.anchored("identity.nonce_regex").fullmatch(nonce) is not None and isinstance(mac, str)
and hmac.compare_digest(challenge_mac(secret, nonce).encode(), mac.encode("ascii")))
except (ValueError, UnicodeError):
return False
def verify_proof(secret: str, nonce: Any, pid: int, port: int, proof: Any) -> bool:
"""The terminator's check of an answer: bound to this nonce, this pid and this port, so a proof for another challenge, process or port fails."""
try:
return (isinstance(nonce, str) and policy.anchored("identity.nonce_regex").fullmatch(nonce) is not None and isinstance(proof, str)
and hmac.compare_digest(surface_proof(secret, nonce, pid, port).encode(), proof.encode("ascii")))
except (ValueError, UnicodeError):
return False
def identity_request(host: str, port: int, nonce: str, mac: str) -> str:
"""The exact request head the terminator sends on the connection it is about to hand to ChatGPT, before the first ChatGPT byte."""
return (f"GET {policy.value('surface.identity_path')} HTTP/1.1\r\nHost: {host}:{port}\r\n{policy.value('surface.identity_challenge_header')}: {nonce}\r\n"
f"{policy.value('surface.identity_mac_header')}: {mac}\r\nConnection: keep-alive\r\n\r\n")
class SurfaceIdentityError(Exception):
"""The listener could not prove it is the agent. `word` is a fixed word (never text from the network)."""
def __init__(self, word: str):
super().__init__(word)
self.word = word
async def prove_listener(reader: asyncio.StreamReader, writer: asyncio.StreamWriter, *, host: str, port: int, secret: str, pid: int,
timeout: float = 5.0) -> None:
"""The terminator's half of the proof: on a freshly opened connection to the surface, challenge it and check its answer; return only when it is the agent
that holds `secret` for `pid` and `port`, with the answer consumed (the connection is kept alive for ChatGPT's bytes). Raises SurfaceIdentityError, and
the caller then closes the connection and sends nothing of ChatGPT's."""
nonce = secrets.token_urlsafe(policy.value("protocol.nonce_bytes"))
try:
writer.write(identity_request(host, port, nonce, challenge_mac(secret, nonce)).encode("ascii"))
await asyncio.wait_for(writer.drain(), timeout)
head = await asyncio.wait_for(reader.readuntil(b"\r\n\r\n"), timeout)
except (OSError, asyncio.TimeoutError, asyncio.IncompleteReadError, asyncio.LimitOverrunError, ValueError):
raise SurfaceIdentityError("no_answer") from None
lines = head[:-4].split(b"\r\n")
if lines[0] != b"HTTP/1.1 200 OK" or len(head) > 2048:
raise SurfaceIdentityError("not_the_agent")
seen: dict[bytes, bytes] = {}
for line in lines[1:]:
name, _, value = line.partition(b":")
if name.lower() in seen:
raise SurfaceIdentityError("not_the_agent")
seen[name.lower()] = value.strip()
if seen.get(b"content-length") != b"0":
raise SurfaceIdentityError("not_the_agent")
if not verify_proof(secret, nonce, pid, port, seen.get(policy.value("surface.identity_proof_header").encode(), b"").decode("latin-1")):
raise SurfaceIdentityError("bad_proof")
# ------------------------------------------------------------------------------------------------------------------------ surfaces.json
@dataclass(frozen=True)
class SurfaceTarget:
pid: int
host: str
port: int
secret: str
started_at: float
def surfaces_path() -> Path:
return state_dir() / SURFACES_FILE
def lane_path() -> Path:
return state_dir() / LANE_FILE
def write_surfaces_file(*, pid: int, port: int, secret: str, path: Path | None = None) -> None:
"""state_dir()/surfaces.json (0600: the secret in it is what proves the listener is the agent). Replaces whatever an earlier start left."""
state_io.atomic_json(path or surfaces_path(), {"v": 1, "pid": pid, "host": LOOPBACK, "port": port, "secret": secret, "started_at": int(time.time())})
def remove_surfaces_file(*, pid: int, secret: str, path: Path | None = None) -> None:
"""Remove surfaces.json, but only while it still names this listener (a newer start may have written its own)."""
path = path or surfaces_path()
try:
data = json.loads(path.read_text(encoding="utf-8"))
if data.get("pid") == pid and data.get("secret") == secret:
path.unlink()
except (OSError, ValueError, AttributeError):
pass
def _pid_alive(pid: int) -> bool:
try:
import psutil
return bool(psutil.pid_exists(pid))
except ImportError:
pass
try:
os.kill(pid, 0)
except ProcessLookupError:
return False
except OSError:
return True
return True
def load_target(path: Path | None = None, *, require_alive: bool = True) -> SurfaceTarget:
"""What the terminator dials: the one validated address of the E2E listener. Raises SurfaceIdentityError("no_surface") for anything that is not a
well-formed surfaces.json of a live agent: a missing or oversized file, a link, one other accounts can read, a bad field, a pid that is gone."""
path = path or surfaces_path()
try:
info = path.lstat()
if not stat.S_ISREG(info.st_mode) or info.st_size > 4096 or (os.name != "nt" and info.st_mode & 0o077):
raise SurfaceIdentityError("no_surface")
data = json.loads(path.read_text(encoding="utf-8"))
except (OSError, ValueError):
raise SurfaceIdentityError("no_surface") from None
try:
pid, port, started = data["pid"], data["port"], data["started_at"]
ok = (isinstance(data, dict) and set(data) == {"v", "pid", "host", "port", "secret", "started_at"} and data["v"] == 1 and data["host"] == LOOPBACK
and type(pid) is int and 1 <= pid <= 0xFFFFFFFF and type(port) is int and 1024 <= port <= 65535
and isinstance(data["secret"], str) and policy.anchored("identity.attach_secret_regex").fullmatch(data["secret"]) is not None
and isinstance(started, (int, float)) and not isinstance(started, bool))
except (KeyError, TypeError):
ok = False
if not ok or (require_alive and not _pid_alive(pid)):
raise SurfaceIdentityError("no_surface")
return SurfaceTarget(pid, LOOPBACK, port, data["secret"], float(started))
# ------------------------------------------------------------------------------------------------------------------------ the wrappers
def _marked(scope: dict, surface: str, **changes: Any) -> dict:
state = {**(scope.get("state") or {}), SURFACE_KEY: surface}
state.pop(E2E_HOST_KEY, None) # only the E2E wrapper writes it, below: whatever came in with the scope is not believed
if "host" in changes:
state[E2E_HOST_KEY] = changes.pop("host")
return {**scope, **changes, "state": state}
class LocalSurface:
"""The LOCAL listener's wrapper: marks every request LOCAL, otherwise changes nothing (the lifespan, which the inner app runs once, passes through)."""
def __init__(self, app):
self.app = app
async def __call__(self, scope, receive, send):
if scope["type"] == "lifespan":
await self.app(scope, receive, send)
return
token = _current.set(LOCAL)
try:
await self.app(_marked(scope, LOCAL), receive, send)
finally:
_current.reset(token)
async def _respond(send, status: int, body: bytes = b"", *, extra=(), head: bool = False) -> None:
headers = [(b"cache-control", b"no-store"), (b"content-length", str(len(body)).encode())]
if body:
headers.append((b"content-type", b"application/json"))
headers.extend(extra)
await send({"type": "http.response.start", "status": status, "headers": headers})
await send({"type": "http.response.body", "body": b"" if head else body})
def _values(scope, name: bytes) -> list[bytes]:
return [value for key, value in scope.get("headers", []) if key == name]
class E2ESurface:
"""The E2E listener's wrapper: everything that arrives here is remote, so it is closed by default and the only things let through are what ChatGPT needs.
(RelaySurface, below, is the same wrapper for the legacy relay's lane: its own mark, its own list of paths, and the Host of its own socket.)
In order: a lifespan or websocket scope never reaches the app; the head is bounded (headers, their size, the whole head, one body length, no Upgrade); the identity
proof answers its reserved path and nothing else does; the request names this computer, once, by its exact name (421); the path is on the list (404, and
`/mcp/` is served as `/mcp`); `/healthz` says only that the agent is up; the headers that claim an origin are removed; the body is capped while it streams
(413); and the request is marked E2E for the authentication and the gate. `host` is the device's full name; `secret`, `pid` and `port` are what surfaces.json
holds."""
surface = E2E
paths = E2E_PATHS
_answers_healthz = True # E2E says only that the agent is up; the relay lane lets the app answer, as the old relay always forwarded it
def __init__(self, app, *, host: str, secret: str, pid: int, port: int):
self.app = app
self.host = host.lower()
self._secret = secret
self._pid = pid
self._port = port
_key(secret) # a secret that cannot key the proof is a start that must fail now, not at the first challenge
self._body_max = policy.value("surface.body_max_bytes")
self._headers_max = policy.value("surface.headers_max")
self._value_max = policy.value("surface.header_value_max_bytes")
self._head_max = head_cap()
self._proof_path = policy.value("surface.identity_path")
async def __call__(self, scope, receive, send):
kind = scope["type"]
if kind == "lifespan": # E2E runs with the lifespan off; if one ever arrives it is answered here, so the inner app's lifespan never runs twice
while True:
message = await receive()
if message["type"] == "lifespan.startup":
await send({"type": "lifespan.startup.complete"})
elif message["type"] == "lifespan.shutdown":
await send({"type": "lifespan.shutdown.complete"})
return
if kind != "http": # a websocket (or anything else): the E2E surface has no scope but http, so nothing skips the authentication
if kind == "websocket":
await receive()
await send({"type": "websocket.close", "code": 1008})
return
headers = scope.get("headers", [])
if (len(headers) > self._headers_max or any(len(value) > self._value_max for _, value in headers)
or len(scope.get("raw_path") or b"") + len(scope.get("query_string") or b"") + sum(len(key) + len(value) + 4 for key, value in headers) > self._head_max):
await _respond(send, 431, b'{"error":"request_header_fields_too_large"}', extra=[(b"connection", b"close")])
return
if _values(scope, b"upgrade"): # no websocket and no h2c upgrade: a request that asks for one is refused, not served as a plain one
await _respond(send, 400, b'{"error":"upgrade_not_supported"}', extra=[(b"connection", b"close")])
return
lengths = _values(scope, b"content-length")
if len(lengths) > 1 or (lengths and not lengths[0].isdigit()) or (lengths and _values(scope, b"transfer-encoding")):
await _respond(send, 400, b'{"error":"invalid_request"}', extra=[(b"connection", b"close")]) # the framing a smuggler plays with
return
path = scope.get("path", "")
if path == self._proof_path:
await self._prove(scope, send)
return
hosts = _values(scope, b"host")
if len(hosts) != 1 or hosts[0].lower() != self.host.encode("latin-1"):
await _respond(send, 421, b'{"error":"misdirected_request"}')
return
if path not in self.paths:
await _respond(send, 404, b'{"error":"not_found"}')
return
if path == "/healthz" and self._answers_healthz:
if scope["method"] not in ("GET", "HEAD"):
await _respond(send, 405, b'{"error":"method_not_allowed"}', extra=[(b"allow", b"GET, HEAD")])
return
await _respond(send, 200, b'{"ok":true}', head=scope["method"] == "HEAD")
return
if lengths and int(lengths[0]) > self._body_max:
await _respond(send, 413, b'{"error":"request_too_large"}', extra=[(b"connection", b"close")])
return
changes: dict[str, Any] = {"headers": [(key, value) for key, value in headers
if not key.startswith(_STRIPPED_PREFIXES) and key not in _STRIPPED_NAMES]}
if path == "/mcp/":
changes["path"], changes["raw_path"] = "/mcp", b"/mcp"
if self.surface == E2E: # only the E2E listener names an encrypted origin; the relay lane's requests belong to the primary one
changes["host"] = self.host
await self._serve(_marked(scope, self.surface, **changes), receive, send)
async def _prove(self, scope, send) -> None:
"""The reserved identity path: answered only to a valid challenge (a 404 like any other path otherwise, so a stranger learns nothing)."""
nonces = _values(scope, policy.value("surface.identity_challenge_header").encode())
macs = _values(scope, policy.value("surface.identity_mac_header").encode())
nonce = nonces[0].decode("latin-1") if len(nonces) == 1 and len(macs) == 1 else ""
if scope["method"] != "GET" or not verify_challenge(self._secret, nonce, macs[0].decode("latin-1") if macs else ""):
await _respond(send, 404, b'{"error":"not_found"}')
return
proof = surface_proof(self._secret, nonce, self._pid, self._port).encode()
await _respond(send, 200, b"", extra=[(policy.value("surface.identity_proof_header").encode(), proof)])
async def _serve(self, scope, receive, send) -> None:
total = 0
exceeded = False
started = False
async def capped_receive():
nonlocal total, exceeded
if exceeded:
return {"type": "http.disconnect"}
message = await receive()
if message["type"] == "http.request":
total += len(message.get("body", b""))
if total > self._body_max: # counted as it streams, whatever Content-Length said (or did not say)
exceeded = True
return {"type": "http.disconnect"}
return message
async def guarded_send(message):
nonlocal started
if exceeded:
return
if message["type"] == "http.response.start":
started = True
await send(message)
token = _current.set(self.surface)
try:
await self.app(scope, capped_receive, guarded_send)
except Exception:
if not exceeded:
raise
finally:
_current.reset(token)
if exceeded and not started:
await _respond(send, 413, b'{"error":"request_too_large"}', extra=[(b"connection", b"close")])
class RelaySurface(E2ESurface):
"""The relay lane's wrapper (WP11): what the legacy relay's client dials. It is the E2E wrapper with the relay's own allowlist (`LANE_PATHS`), the mark RELAY, and one
Host: the lane's own socket, `127.0.0.1:<port>`, the only name the client writes (the relay's Host is dropped and the client's HTTP library sets this one); any other
Host is a request that did not come from the client (421). The app answers `/healthz` itself here: the installers and the site read the old relay's health through it,
and that report is the same one 1.1 gave. What this surface never carries, wherever a request says it came from: the routes of the computer's own tools, the
bootstrap bearer (middleware.AuthMiddleware), the paywall gate (the relay gates what it serves) and a websocket."""
surface = RELAY
paths = LANE_PATHS
_answers_healthz = False
def __init__(self, app, *, secret: str, pid: int, port: int):
super().__init__(app, host=f"{LOOPBACK}:{port}", secret=secret, pid=pid, port=port)
# ------------------------------------------------------------------------------------------------------------------------ the listeners
def head_cap() -> int:
"""The most a request head may weigh: one header as long as the policy allows, plus an ordinary head. h11 enforces it only on a head that arrives incomplete (a
head that comes whole in one read is parsed whatever its size), so the E2E wrapper counts it again."""
return policy.value("surface.header_value_max_bytes") + policy.value("terminator.request_head_max_bytes")
def uvicorn_config(app, *, log_level: str, concurrency: int, ws: str = "auto", lifespan: str = "auto", **options: Any):
"""The settings both servers share (uvicorn is imported here so that importing this module stays cheap). `proxy_headers` is OFF and no address is trusted to
forward: uvicorn would otherwise believe X-Forwarded-For from 127.0.0.1, which is every caller there is (the tunnel and the terminator are on loopback), and the
rate-limit key and the client of a request would be whatever the remote caller wrote. Access logs are off (the request line carries OAuth and pairing
parameters)."""
import uvicorn
return uvicorn.Config(app, interface="asgi3", http="h11", log_level=log_level, access_log=False, proxy_headers=False, forwarded_allow_ips="", ws=ws, lifespan=lifespan,
timeout_keep_alive=policy.value("surface.keep_alive_s"), limit_concurrency=concurrency,
h11_max_incomplete_event_size=head_cap(), **options)
def _local_server_class():
import uvicorn
class LocalServer(uvicorn.Server):
"""The LOCAL server. When it starts to shut down (a signal) it first tells the runner, so the E2E listener and surfaces.json go before the process does:
uvicorn raises the signal again as soon as `serve` is done, which ends the process before anything after it could run."""
def __init__(self, config, before_shutdown: Callable[[], Any]):
super().__init__(config)
self._before_shutdown = before_shutdown
async def shutdown(self, sockets=None):
try:
await self._before_shutdown()
except Exception: # noqa: BLE001
_LOG.warning("e2e surface did not stop cleanly", exc_info=False)
await super().shutdown(sockets=sockets)
return LocalServer
def _server_class():
import uvicorn
class QuietServer(uvicorn.Server):
"""The E2E server: it does not take over SIGINT and SIGTERM (the LOCAL server owns them, and stops this one)."""
@contextlib.contextmanager
def capture_signals(self):
yield
return QuietServer
def _bind_loopback() -> socket.socket:
"""A listening socket on 127.0.0.1 and an ephemeral port. On Windows it is bound exclusively (SO_EXCLUSIVEADDRUSE), so nothing can share the port."""
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
if hasattr(socket, "SO_EXCLUSIVEADDRUSE"):
sock.setsockopt(socket.SOL_SOCKET, socket.SO_EXCLUSIVEADDRUSE, 1)
sock.bind((LOOPBACK, 0))
sock.listen(socket.SOMAXCONN)
sock.setblocking(False)
except BaseException:
sock.close()
raise
return sock
@dataclass
class _Listener:
config: E2EConfig | None # what the E2E listener serves; None for the relay lane, which has nothing to follow
server: Any
task: "asyncio.Task[None]"
pid: int
port: int
secret: str
class SurfaceRunner:
"""Serves the app on the LOCAL surface and, while they are wanted, on the E2E one and on the relay lane: up to three uvicorn servers in one loop. `values`, `bind`
and `server_class` exist so the tests can drive it without a runtime.env, a real port or a real uvicorn; the defaults are the real ones (`lane_server_class` and
`lane_path` are the lane's own, so a test that breaks the E2E server does not break the lane with it)."""
def __init__(self, app, *, port: int, log_level: str = "info", values: Callable[[], dict[str, str]] = runtime_values,
bind: Callable[[], socket.socket] = _bind_loopback, server_class: Any = None, state_path: Path | None = None, watch: float = WATCH_SECONDS,
lane_path: Path | None = None, lane_server_class: Any = None):
self.app = app
self.port = port
self.log_level = log_level
self._values = values
self._bind = bind
self._server_class = server_class
self._path = state_path
self._watch_seconds = watch
self._lane_file = lane_path
self._lane_server_class = lane_server_class
self._listener: _Listener | None = None
self._failures = 0
self._retry_at = 0.0
self._lane: _Listener | None = None
self._lane_failures = 0
self._lane_retry_at = 0.0
self._stopping = False
self.local: Any = None
# ---- the whole thing
async def serve(self) -> None:
"""Block until the LOCAL server stops (a signal, or it could not start: then SystemExit, as `uvicorn.run` had it)."""
self.local = self.make_local()
watcher = asyncio.create_task(self._watch())
try:
await self.local.serve()
finally:
watcher.cancel()
await asyncio.gather(watcher, return_exceptions=True)
await self.stop_lane()
await self.stop_e2e()
def make_local(self):
config = uvicorn_config(LocalSurface(self.app), log_level=self.log_level, concurrency=LOCAL_CONCURRENCY, host=LOOPBACK, port=self.port)
return _local_server_class()(config, self._begin_shutdown)
async def _begin_shutdown(self) -> None:
self._stopping = True
await self.stop_lane()
await self.stop_e2e()
async def _watch(self) -> None:
while True:
try:
await self.reconcile()
except Exception: # noqa: BLE001 - nothing the E2E side does may take the agent down
_LOG.warning("surface supervisor error", exc_info=False)
await asyncio.sleep(self._watch_seconds)
# ---- the E2E listener
async def reconcile(self) -> None:
"""Make the E2E listener and the relay lane match what is wanted now. Each has its own boundary: one that fails never keeps the other from being looked at."""
if self._stopping:
return
try:
await self._reconcile_e2e()
finally:
await self._reconcile_lane()
async def _reconcile_e2e(self) -> None:
"""Make the E2E listener match what is wanted now: start it, stop it, or start it again after it died (with a backoff)."""
wanted = e2e_config(self._values())
listener = self._listener
if listener is not None and listener.task.done():
_LOG.warning("e2e surface stopped by itself")
await self.stop_e2e()
self._failures += 1
self._retry_at = time.monotonic() + min(policy.value("timing.backoff_max_s"), policy.value("timing.backoff_min_s") * 2 ** (self._failures - 1))
listener = None
elif listener is not None:
self._failures = 0
if listener is not None and listener.config != wanted:
await self.stop_e2e()
listener = None
if wanted is not None and listener is None and time.monotonic() >= self._retry_at:
try:
await self.start_e2e(wanted)
except Exception: # noqa: BLE001 - one boundary: a listener that cannot start is a missing remote surface, not a dead agent
_LOG.warning("e2e surface could not start", exc_info=False)
await self.stop_e2e()
self._failures += 1
self._retry_at = time.monotonic() + min(policy.value("timing.backoff_max_s"), policy.value("timing.backoff_min_s") * 2 ** (self._failures - 1))
async def start_e2e(self, wanted: E2EConfig) -> None:
sock = self._bind()
try:
port = sock.getsockname()[1]
secret = new_secret()
pid = os.getpid()
surface = E2ESurface(self.app, host=wanted.host, secret=secret, pid=pid, port=port)
config = uvicorn_config(surface, log_level=self.log_level, concurrency=policy.value("terminator.max_streams") * E2E_CONCURRENCY_FACTOR,
lifespan="off", ws="none", timeout_graceful_shutdown=GRACEFUL_SHUTDOWN_SECONDS)
server = (self._server_class or _server_class())(config)
task = asyncio.create_task(self._guard(server, sock))
except BaseException:
sock.close()
raise
self._listener = _Listener(wanted, server, task, pid, port, secret)
deadline = time.monotonic() + START_TIMEOUT_SECONDS
while not getattr(server, "started", False) and not task.done():
if time.monotonic() > deadline:
raise RuntimeError("e2e surface did not start")
await asyncio.sleep(0.02)
if task.done() or self._stopping or self._listener is None:
raise RuntimeError("e2e surface did not start")
_allow_everywhere(wanted.host) # a computer that enrolled while the agent ran: the MCP transport must accept the name too
write_surfaces_file(pid=pid, port=port, secret=secret, path=self._path)
_serving["e2e"] = True
# ---- the relay lane
async def _reconcile_lane(self) -> None:
"""The same loop for the lane the legacy relay's client dials (WP11): up while `relay_lane_wanted`, down when it is not, up again after it died (with the
backoff of the E2E listener). It needs no label: a computer that has never enrolled is exactly who the old relay serves."""
if self._stopping:
return
wanted = relay_lane_wanted(self._values())
listener = self._lane
if listener is not None and listener.task.done():
_LOG.warning("relay lane stopped by itself")
await self.stop_lane()
self._lane_failures += 1
self._lane_retry_at = time.monotonic() + min(policy.value("timing.backoff_max_s"), policy.value("timing.backoff_min_s") * 2 ** (self._lane_failures - 1))
listener = None
elif listener is not None:
self._lane_failures = 0
if listener is not None and not wanted:
await self.stop_lane()
listener = None
if wanted and listener is None and time.monotonic() >= self._lane_retry_at:
try:
await self.start_lane()
except Exception: # noqa: BLE001 - one boundary: a lane that cannot start is a legacy route that answers "unavailable" and is retried, not a dead agent
_LOG.warning("relay lane could not start", exc_info=False)
await self.stop_lane()
self._lane_failures += 1
self._lane_retry_at = time.monotonic() + min(policy.value("timing.backoff_max_s"),
policy.value("timing.backoff_min_s") * 2 ** (self._lane_failures - 1))
async def start_lane(self) -> None:
sock = self._bind()
try:
port = sock.getsockname()[1]
secret = new_secret()
pid = os.getpid()
surface = RelaySurface(self.app, secret=secret, pid=pid, port=port)
config = uvicorn_config(surface, log_level=self.log_level, concurrency=policy.value("terminator.max_streams") * E2E_CONCURRENCY_FACTOR,
lifespan="off", ws="none", timeout_graceful_shutdown=GRACEFUL_SHUTDOWN_SECONDS)
server = (self._lane_server_class or _server_class())(config)
task = asyncio.create_task(self._guard(server, sock))
except BaseException:
sock.close()
raise
self._lane = _Listener(None, server, task, pid, port, secret)
deadline = time.monotonic() + START_TIMEOUT_SECONDS
while not getattr(server, "started", False) and not task.done():
if time.monotonic() > deadline:
raise RuntimeError("relay lane did not start")
await asyncio.sleep(0.02)
if task.done() or self._stopping or self._lane is None:
raise RuntimeError("relay lane did not start")
write_surfaces_file(pid=pid, port=port, secret=secret, path=self._lane_file or lane_path())
async def stop_lane(self) -> None:
listener, self._lane = self._lane, None
if listener is None:
return
remove_surfaces_file(pid=listener.pid, secret=listener.secret, path=self._lane_file or lane_path()) # first: the client must not dial a port that is going away
await self._stop(listener)
async def _stop(self, listener: _Listener) -> None:
listener.server.should_exit = True
done, _ = await asyncio.wait({listener.task}, timeout=GRACEFUL_SHUTDOWN_SECONDS + 5)
if not done:
listener.server.force_exit = True
listener.task.cancel()
await asyncio.gather(listener.task, return_exceptions=True)
async def _guard(self, server, sock: socket.socket) -> None:
"""The E2E server's boundary. uvicorn exits with SystemExit when it cannot start, and asyncio would carry that out of the event loop and end the agent."""
try:
await server.serve(sockets=[sock])
except (Exception, SystemExit) as exc: # noqa: BLE001
_LOG.warning("e2e surface failed: %s", type(exc).__name__)
finally:
# A server that died before its own shutdown (cancelled, crashed) still has its listening socket registered with the loop: close the asyncio servers
# first, or the next listener that is given the same file descriptor number would be served by this dead one.
for listening in getattr(server, "servers", None) or []:
listening.close()
sock.close()
async def stop_e2e(self) -> None:
listener, self._listener = self._listener, None
_serving["e2e"] = False
if listener is None:
return
remove_surfaces_file(pid=listener.pid, secret=listener.secret, path=self._path) # first: nothing may dial a port that is going away
await self._stop(listener)
def run(app, *, port: int, log_level: str = "info") -> None:
"""`uvicorn.run(app, host="127.0.0.1", port=port)` of the agent, now with the two surfaces. Blocks."""
asyncio.run(SurfaceRunner(app, port=port, log_level=log_level).serve())
files/src/all_mcp/tls_policy.py 985 lines · 51.3 KB
"""The TLS policy of the ALL 2.0 terminator and the proof of what it can honestly claim (plan sections 2, 3 and 7; WP16).
Three things live here, and nothing else in the device decides them:
* **The server context.** TLS 1.3 at least, ALPN ``http/1.1`` (the two words come from ``policy.terminator``), the certificate the generation names. The terminator
refuses to start in exactly one case: the Python has no TLS 1.3 (``TlsUnavailable``, closed word ``tls13_required``). That is a reported ``degraded`` state
(e2e_status, the tray, the heartbeat), never a silent fallback. The OpenSSL version does not stop anything: it is reported per operating system and it gates
ONE claim, the hybrid key exchange (OpenSSL 3.5 brings X25519MLKEM768; an older one serves X25519 and says nothing about hybrid).
* **A parser of the TLS hellos that cross the wire in the clear** (``parse_client_hello``, ``observe_server``): version, cipher suite and the group the server
selected, read from the ServerHello bytes themselves, so a node observer, a pcap and the blindness gate (WP23) all read the same thing, and nothing depends on
what Python's ``ssl`` module is willing to tell. It follows a hello across records and across TCP segments, a HelloRetryRequest and an alert; malformed bytes are
``TlsParseError`` with a closed word, never an ``IndexError``.
* **The capability report** (``capability_report``): real loopback handshakes against the real server context (a client that offers hybrid, one that does not,
one that speaks only TLS 1.2), and from what they showed, exactly which claims this operating system can prove. It is the JSON that feeds ``e2e-proof.json`` and the
TLS sentences of the site (WP27): a sentence that is not in ``allowed_claims`` is not written.
``OPENSSL_CONF`` (the installer's OpenSSL config file) restricts the key-exchange groups and orders the TLS 1.3 suites. It was evaluated against the relay process and is
OFF by default: it is a process-wide setting, and a restricted list refuses a peer that offers only a group left out (a test shows one). It can be
turned on for the supervised e2e child alone (``runtime.env`` word ``ALL_E2E_OPENSSL_CONF=on``, ``apply_openssl_conf``), never for the relay process, and a runtime that
ignores it is detected (``openssl_conf_effective``) instead of trusted.
python -m all_mcp.tls_policy report [--out FILE] [--no-probe] # the per-OS report of this computer, JSON on stdout
"""
from __future__ import annotations
import argparse
import contextlib
import datetime
import json
import logging
import os
import platform
import queue
import ssl
import struct
import subprocess
import sys
import tempfile
import threading
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Callable, Mapping, Sequence
from . import e2e_policy as policy
from .windows_console import hide_child_consoles, no_window_flags
log = logging.getLogger("all_mcp.tls_policy")
REPORT_VERSION = 1
TLS_13, TLS_12 = 0x0304, 0x0303
# Registry values (IANA TLS parameters), not policy: what a hello carries on the wire.
GROUP_X25519, GROUP_SECP256R1, GROUP_SECP384R1, GROUP_SECP521R1 = 0x001D, 0x0017, 0x0018, 0x0019
GROUP_X25519_MLKEM768, GROUP_SECP256R1_MLKEM768, GROUP_X25519_KYBER768_DRAFT = 0x11EC, 0x11EB, 0x6399
HYBRID_GROUPS = frozenset({GROUP_X25519_MLKEM768, GROUP_SECP256R1_MLKEM768, GROUP_X25519_KYBER768_DRAFT})
GROUP_NAMES = {
GROUP_X25519: "X25519", GROUP_SECP256R1: "secp256r1", GROUP_SECP384R1: "secp384r1", GROUP_SECP521R1: "secp521r1", 0x001E: "X448",
0x0100: "ffdhe2048", 0x0101: "ffdhe3072", 0x0102: "ffdhe4096", GROUP_X25519_MLKEM768: "X25519MLKEM768", GROUP_SECP256R1_MLKEM768: "SecP256r1MLKEM768",
GROUP_X25519_KYBER768_DRAFT: "X25519Kyber768Draft00",
}
CIPHER_NAMES = {
0x1301: "TLS_AES_128_GCM_SHA256", 0x1302: "TLS_AES_256_GCM_SHA384", 0x1303: "TLS_CHACHA20_POLY1305_SHA256", 0x1304: "TLS_AES_128_CCM_SHA256",
0x1305: "TLS_AES_128_CCM_8_SHA256",
}
ALERT_NAMES = {
0: "close_notify", 10: "unexpected_message", 20: "bad_record_mac", 40: "handshake_failure", 42: "bad_certificate", 47: "illegal_parameter",
70: "protocol_version", 71: "insufficient_security", 80: "internal_error", 86: "inappropriate_fallback", 90: "user_canceled", 109: "missing_extension",
112: "unrecognized_name", 120: "no_application_protocol",
}
VERSION_NAMES = {0x0304: "TLSv1.3", 0x0303: "TLSv1.2", 0x0302: "TLSv1.1", 0x0301: "TLSv1.0", 0x0300: "SSLv3"}
HELLO_RETRY_RANDOM = bytes.fromhex("cf21ad74e59a6111be1d8c021e65b891c2a211167abb8c5e079e09e2c8a8339c") # RFC 8446 section 4.1.3: SHA-256("HelloRetryRequest")
EXT_SERVER_NAME, EXT_SUPPORTED_GROUPS, EXT_ALPN, EXT_SUPPORTED_VERSIONS, EXT_KEY_SHARE, EXT_ECH = 0, 10, 16, 43, 51, 0xFE0D
RECORD_HANDSHAKE, RECORD_ALERT, RECORD_CCS, RECORD_APPLICATION = 22, 21, 20, 23
RECORD_HEADER_BYTES = 5
RECORD_MAX_BYTES = 16384 + 2048 # a handshake record is at most 2^14 (the field allows more only for ciphertext); the slack is for sloppy senders
HELLO_MAX_BYTES = 1 << 17 # the largest hello that is read: a ClientHello offering X25519MLKEM768 is about 1.5 KB, 128 KB is already abuse
# The OpenSSL that brings X25519MLKEM768 in its default groups; the one number of this file that is a fact about a library, not a choice of ours.
HYBRID_MIN_OPENSSL = (3, 5)
PROBE_NAME = "probe.all.invalid"
PROBE_SECONDS = 20.0
PROBE_ROUNDS = 16
# The claims this module can prove, in the order the report lists them, and the one sentence of evidence each needs. A claim not in this table is never written.
CLAIMS = {
"tls_1_3_minimum": "the terminator's context refuses below TLS 1.3 and a loopback handshake negotiated TLS 1.3",
"tls_1_2_refused": "a client that speaks only TLS 1.2 was refused with an alert",
"alpn_http_1_1": "a client that offered http/1.1 was given http/1.1",
"forward_secrecy": "the ServerHello selected an ephemeral key-exchange group (TLS 1.3 has no static one)",
"classical_fallback": "a client that offers no hybrid group still completed the handshake (X25519 or another classical group)",
"hybrid_x25519mlkem768": "OpenSSL 3.5 or later AND a client that offered X25519MLKEM768 was answered with X25519MLKEM768",
}
# What no loopback report can prove, with the reason (the plan's section 10): they wait for the gate's real captures or are never true.
NEVER_CLAIMED = {
"post_quantum_with_chatgpt": "needs the node counters of a real ChatGPT that negotiated 0x11EC (G1); a loopback client is not ChatGPT",
"aes_256_only": "the client chooses the suite: ChatGPT may pick AES-128-GCM or ChaCha20",
"tls_1_3_always": "only what the gate proves per system; the other end decides what it offers",
"ech": "no Encrypted Client Hello: the node reads the SNI by design",
}
STATES = ("ready", "degraded")
# The switch of the OPENSSL_CONF restriction: a word of runtime.env that only the supervised e2e child honours. Absent means off.
OPENSSL_CONF_SWITCH = "ALL_E2E_OPENSSL_CONF"
OPENSSL_CONF_FILE = "openssl-all.cnf"
OPENSSL_CONF_DEFAULT_ON = False
ON_WORDS = frozenset({"on", "1", "true", "yes"})
# The groups the restricted list offers: hybrid first where the library knows it ("?" skips a name an older OpenSSL does not know), then the classical groups every
# TLS 1.3 peer of ours speaks. The TLS 1.3 suites are the three every OpenSSL has, ChaCha20 last so that a machine with AES hardware keeps its choice.
OPENSSL_CONF_GROUPS = "?X25519MLKEM768:X25519:secp256r1:secp384r1"
OPENSSL_CONF_SUITES = "TLS_AES_256_GCM_SHA384:TLS_AES_128_GCM_SHA256:TLS_CHACHA20_POLY1305_SHA256"
class TlsParseError(ValueError):
"""The bytes are not a TLS hello. ``word`` is a closed word (``bad_record``, ``bad_hello``, ``bad_extension``, ``too_large``, ``not_handshake``)."""
def __init__(self, word: str):
super().__init__(word)
self.word = word
class TlsUnavailable(RuntimeError):
"""The terminator cannot serve: this Python has no TLS 1.3. ``word`` is what the status file and the heartbeat carry."""
def __init__(self, word: str = "tls13_required"):
super().__init__(word)
self.word = word
# ------------------------------------------------------------------------------------------------------------------------ names
def is_grease(value: int) -> bool:
return (value & 0x0F0F) == 0x0A0A and (value >> 8) == (value & 0xFF)
def group_name(group: int | None) -> str | None:
if group is None:
return None
return "grease" if is_grease(group) else GROUP_NAMES.get(group, f"0x{group:04x}")
def cipher_name(cipher: int | None) -> str | None:
if cipher is None:
return None
return "grease" if is_grease(cipher) else CIPHER_NAMES.get(cipher, f"0x{cipher:04x}")
def version_name(version: int | None) -> str | None:
return None if version is None else VERSION_NAMES.get(version, f"0x{version:04x}")
# ------------------------------------------------------------------------------------------------------------------------ the hello parser
@dataclass(frozen=True)
class ClientHelloInfo:
"""What a ClientHello says in the clear. ``key_share_groups`` are the groups it actually sent a key for (the server can choose only among those without a retry)."""
versions: tuple[int, ...]
cipher_suites: tuple[int, ...]
groups: tuple[int, ...]
key_share_groups: tuple[int, ...]
server_name: str | None
alpn: tuple[str, ...]
extensions: tuple[int, ...]
records: int
@property
def offers_tls13(self) -> bool:
return TLS_13 in self.versions
@property
def offers_hybrid(self) -> bool:
return any(group in HYBRID_GROUPS for group in self.groups)
@property
def sends_hybrid_key_share(self) -> bool:
return any(group in HYBRID_GROUPS for group in self.key_share_groups)
def as_dict(self) -> dict[str, Any]:
return {"versions": [version_name(v) for v in self.versions if not is_grease(v)], "offers_hybrid": self.offers_hybrid,
"sends_hybrid_key_share": self.sends_hybrid_key_share, "groups": [group_name(g) for g in self.groups if not is_grease(g)],
"key_share_groups": [group_name(g) for g in self.key_share_groups if not is_grease(g)],
"cipher_suites": [cipher_name(c) for c in self.cipher_suites if not is_grease(c)], "server_name": self.server_name, "alpn": list(self.alpn),
"ech": EXT_ECH in self.extensions, "records": self.records}
@dataclass(frozen=True)
class ServerHelloInfo:
"""The ServerHello (or the HelloRetryRequest): the negotiated version, the suite and the group the server selected. In TLS 1.3 the version is the
``supported_versions`` extension, the legacy field of the hello says 1.2 for every handshake; ``group`` is None for TLS 1.2 (its curve is in a later message)."""
version: int
cipher: int
group: int | None
hello_retry_request: bool
extensions: tuple[int, ...]
records: int
@property
def version_name(self) -> str | None:
return version_name(self.version)
@property
def cipher_name(self) -> str | None:
return cipher_name(self.cipher)
@property
def group_name(self) -> str | None:
return group_name(self.group)
@property
def hybrid(self) -> bool:
return self.group in HYBRID_GROUPS
def as_dict(self) -> dict[str, Any]:
return {"version": self.version_name, "cipher": self.cipher_name, "group": self.group_name, "group_id": self.group,
"hybrid": self.hybrid, "hello_retry_request": self.hello_retry_request}
@dataclass(frozen=True)
class TlsAlert:
level: int
description: int
@property
def name(self) -> str:
return ALERT_NAMES.get(self.description, f"alert_{self.description}")
@dataclass(frozen=True)
class ServerObservation:
"""What the server side of a connection said before it went encrypted: the hellos (a retry request, then the real one), an alert if it refused, and whether
the stream held enough to be sure (``complete`` is False while a record or a message is cut: read more bytes and ask again)."""
hellos: tuple[ServerHelloInfo, ...] = ()
alert: TlsAlert | None = None
complete: bool = False
@property
def negotiated(self) -> ServerHelloInfo | None:
"""The hello that settled the handshake: the last one that is not a retry request (or the retry request when nothing followed it)."""
real = [hello for hello in self.hellos if not hello.hello_retry_request]
return real[-1] if real else (self.hellos[-1] if self.hellos else None)
def _read_records(stream: bytes) -> list[tuple[int, bytes]]:
"""The whole TLS records at the start of `stream` as (type, body); a record that is still arriving is left out. Stops at the first encrypted record: everything
after the hello of a TLS 1.3 handshake is ciphertext."""
records: list[tuple[int, bytes]] = []
offset = 0
while offset + RECORD_HEADER_BYTES <= len(stream):
kind, major, length = stream[offset], stream[offset + 1], struct.unpack_from(">H", stream, offset + 3)[0]
if kind not in (RECORD_HANDSHAKE, RECORD_ALERT, RECORD_CCS, RECORD_APPLICATION) or major != 3 or length > RECORD_MAX_BYTES or length == 0:
raise TlsParseError("bad_record")
if offset + RECORD_HEADER_BYTES + length > len(stream):
break
records.append((kind, stream[offset + RECORD_HEADER_BYTES:offset + RECORD_HEADER_BYTES + length]))
offset += RECORD_HEADER_BYTES + length
if kind == RECORD_APPLICATION:
break
return records
def _handshake_messages(stream: bytes, *, server: bool = False) -> tuple[list[tuple[int, bytes, int]], TlsAlert | None, bool]:
"""The handshake messages (type, body, records they span) of the clear part of `stream`, rebuilt across records: one message may be cut into several records and
one record may hold several messages. Returns them with the alert (if a record was an alert) and whether the last message is whole.
The clear part ends at the first change_cipher_spec: after it a TLS 1.2 handshake is encrypted (its Finished is still a "handshake" record), and a TLS 1.3 client
or server sends it only for middleboxes, followed by ciphertext. The one exception is a server that answered with a HelloRetryRequest: its real ServerHello comes
after that record, in the clear, once the client has sent its second hello."""
messages: list[tuple[int, bytes, int]] = []
pending = bytearray()
spans = 0
alert: TlsAlert | None = None
for kind, body in _read_records(stream):
if kind == RECORD_ALERT:
if len(body) >= 2 and alert is None:
alert = TlsAlert(body[0], body[1])
continue
if kind == RECORD_CCS:
if server and not pending and messages and messages[-1][0] == 2 and messages[-1][1][2:34] == HELLO_RETRY_RANDOM:
continue
break
if kind != RECORD_HANDSHAKE:
break
pending += body
spans += 1
while len(pending) >= 4:
if server and pending[0] != 2: # what follows a ServerHello in TLS 1.2 (a certificate chain, say) is not read: it may be large and it is not ours
messages.append((pending[0], b"", spans))
return messages, alert, True
size = int.from_bytes(pending[1:4], "big")
if size > HELLO_MAX_BYTES:
raise TlsParseError("too_large")
if len(pending) < 4 + size:
break
messages.append((pending[0], bytes(pending[4:4 + size]), spans))
del pending[:4 + size]
spans = 1 if pending else 0
return messages, alert, not pending
def _take(data: bytes, offset: int, size: int) -> tuple[bytes, int]:
if size < 0 or offset + size > len(data):
raise TlsParseError("bad_hello")
return data[offset:offset + size], offset + size
def _extensions(block: bytes) -> list[tuple[int, bytes]]:
out: list[tuple[int, bytes]] = []
offset = 0
while offset < len(block):
header, offset = _take(block, offset, 4)
kind, size = struct.unpack(">HH", header)
data, offset = _take(block, offset, size)
out.append((kind, data))
return out
def _u16_list(data: bytes, *, width: int = 2) -> tuple[int, ...]:
"""A length-prefixed list of 16-bit values (the prefix is `width` bytes: 1 for supported_versions, 2 for supported_groups)."""
if len(data) < width or int.from_bytes(data[:width], "big") != len(data) - width or (len(data) - width) % 2:
raise TlsParseError("bad_extension")
body = data[width:]
return struct.unpack(">" + "H" * (len(body) // 2), body)
def _hello_prefix(body: bytes) -> tuple[int, bytes, int]:
"""(legacy version, random, offset after the session id) of a Client or Server hello."""
head, offset = _take(body, 0, 34)
version, random = struct.unpack(">H", head[:2])[0], head[2:34]
size = _take(body, offset, 1)[0][0]
_, offset = _take(body, offset + 1, size)
return version, random, offset
def _client_hello(body: bytes, records: int) -> ClientHelloInfo:
_, _, offset = _hello_prefix(body)
size, offset = _take(body, offset, 2)
suites_raw, offset = _take(body, offset, struct.unpack(">H", size)[0])
if len(suites_raw) % 2:
raise TlsParseError("bad_hello")
skip, offset = _take(body, offset, 1)
_, offset = _take(body, offset, skip[0])
versions: tuple[int, ...] = ()
groups: tuple[int, ...] = ()
shares: list[int] = []
name: str | None = None
alpn: list[str] = []
kinds: list[int] = []
if offset < len(body):
size, offset = _take(body, offset, 2)
block, offset = _take(body, offset, struct.unpack(">H", size)[0])
for kind, data in _extensions(block):
kinds.append(kind)
if kind == EXT_SUPPORTED_VERSIONS:
versions = _u16_list(data, width=1)
elif kind == EXT_SUPPORTED_GROUPS:
groups = _u16_list(data)
elif kind == EXT_KEY_SHARE:
if len(data) < 2 or struct.unpack_from(">H", data, 0)[0] != len(data) - 2:
raise TlsParseError("bad_extension")
position = 2
while position < len(data):
entry, position = _take(data, position, 4)
group, key_size = struct.unpack(">HH", entry)
_, position = _take(data, position, key_size)
shares.append(group)
elif kind == EXT_SERVER_NAME:
if len(data) >= 5 and data[2] == 0:
raw, _ = _take(data, 5, struct.unpack_from(">H", data, 3)[0])
name = raw.decode("ascii", errors="replace")
elif kind == EXT_ALPN:
if len(data) < 2 or struct.unpack_from(">H", data, 0)[0] != len(data) - 2:
raise TlsParseError("bad_extension")
position = 2
while position < len(data):
item, position = _take(data, position + 1, data[position])
alpn.append(item.decode("ascii", errors="replace"))
return ClientHelloInfo(versions=versions, cipher_suites=struct.unpack(">" + "H" * (len(suites_raw) // 2), suites_raw), groups=groups,
key_share_groups=tuple(shares), server_name=name, alpn=tuple(alpn), extensions=tuple(kinds), records=records)
def _server_hello(body: bytes, records: int) -> ServerHelloInfo:
legacy, random, offset = _hello_prefix(body)
head, offset = _take(body, offset, 3)
cipher = struct.unpack(">H", head[:2])[0]
version, group = legacy, None
kinds: list[int] = []
retry = random == HELLO_RETRY_RANDOM
if offset < len(body):
size, offset = _take(body, offset, 2)
block, offset = _take(body, offset, struct.unpack(">H", size)[0])
for kind, data in _extensions(block):
kinds.append(kind)
if kind == EXT_SUPPORTED_VERSIONS:
if len(data) != 2:
raise TlsParseError("bad_extension")
version = struct.unpack(">H", data)[0]
elif kind == EXT_KEY_SHARE:
# ServerHello: a KeyShareEntry (group, length, key); HelloRetryRequest: only the group it wants
if len(data) < 2 or (not retry and len(data) < 4):
raise TlsParseError("bad_extension")
group = struct.unpack_from(">H", data, 0)[0]
if not retry and struct.unpack_from(">H", data, 2)[0] != len(data) - 4:
raise TlsParseError("bad_extension")
return ServerHelloInfo(version=version, cipher=cipher, group=group, hello_retry_request=retry, extensions=tuple(kinds), records=records)
def parse_client_hello(stream: bytes) -> ClientHelloInfo | None:
"""The ClientHello that starts `stream` (the client's bytes of one TCP connection, reassembled), or None while it is not whole yet. A hello may be cut across any
number of records and segments; anything that is not a ClientHello is ``TlsParseError``."""
messages, _alert, _clean = _handshake_messages(stream)
if not messages:
if len(stream) >= RECORD_HEADER_BYTES and stream[0] != RECORD_HANDSHAKE:
raise TlsParseError("not_handshake")
return None
kind, body, records = messages[0]
if kind != 1:
raise TlsParseError("not_handshake")
return _client_hello(body, records)
def observe_server(stream: bytes) -> ServerObservation:
"""What the server's bytes of one connection say in the clear: its ServerHello (and a HelloRetryRequest before it), or the alert it answered with."""
messages, alert, clean = _handshake_messages(stream, server=True)
hellos = tuple(_server_hello(body, records) for kind, body, records in messages if kind == 2)
if messages and messages[0][0] != 2:
raise TlsParseError("not_handshake")
if not messages and alert is None and len(stream) >= RECORD_HEADER_BYTES and stream[0] not in (RECORD_HANDSHAKE, RECORD_ALERT):
raise TlsParseError("not_handshake")
return ServerObservation(hellos=hellos, alert=alert, complete=bool(hellos or alert) and clean)
def parse_server_hello(stream: bytes) -> ServerHelloInfo | None:
"""The ServerHello that settled the handshake (see ``ServerObservation.negotiated``), or None while it is not whole or when the server answered with an alert."""
return observe_server(stream).negotiated
def summarize(hellos: Sequence[ServerHelloInfo]) -> dict[str, dict[str, int]]:
"""Counts of version, suite and group over many ServerHellos (a capture, a node's counters): the three tables ``e2e-proof.json`` carries."""
out: dict[str, dict[str, int]] = {"versions": {}, "ciphers": {}, "groups": {}}
for hello in hellos:
if hello.hello_retry_request:
continue
for table, word in (("versions", hello.version_name), ("ciphers", hello.cipher_name), ("groups", hello.group_name or "none")):
out[table][word or "unknown"] = out[table].get(word or "unknown", 0) + 1
return out
# ------------------------------------------------------------------------------------------------------------------------ the runtime and the server context
@dataclass(frozen=True)
class Runtime:
"""What this Python's TLS library is: reported per operating system, tying the hybrid claim to the OpenSSL that carries it."""
openssl: str
openssl_info: tuple[int, ...]
flavor: str
has_tls13: bool
python: str
system: str
release: str
machine: str
@classmethod
def current(cls) -> "Runtime":
text = getattr(ssl, "OPENSSL_VERSION", "")
flavor = "libressl" if "LibreSSL" in text else "openssl" if text.startswith("OpenSSL") else "other"
has13 = bool(getattr(ssl, "HAS_TLSv1_3", False)) and hasattr(ssl.TLSVersion, "TLSv1_3")
return cls(openssl=text, openssl_info=tuple(getattr(ssl, "OPENSSL_VERSION_INFO", ())[:3]), flavor=flavor, has_tls13=has13,
python=platform.python_version(), system=platform.system(), release=platform.release(), machine=platform.machine())
@property
def hybrid_capable(self) -> bool:
"""OpenSSL 3.5 or later: the only library family that ships X25519MLKEM768 in its defaults. This is the gate of the hybrid claim, nothing more."""
return self.flavor == "openssl" and tuple(self.openssl_info[:2]) >= HYBRID_MIN_OPENSSL
def as_dict(self) -> dict[str, Any]:
return {"os": {"system": self.system, "release": self.release, "machine": self.machine}, "python": self.python,
"openssl": {"version": self.openssl, "version_info": list(self.openssl_info), "flavor": self.flavor, "has_tls13": self.has_tls13,
"hybrid_capable": self.hybrid_capable}}
def tls13_available(runtime: Runtime | None = None) -> bool:
return (runtime or Runtime.current()).has_tls13
def require_tls13(runtime: Runtime | None = None) -> None:
"""The one condition the terminator refuses to start on. The caller reports it (status ``degraded``, reason ``tls13_required``); this is never swallowed."""
if not tls13_available(runtime):
log.error("this Python's TLS library has no TLS 1.3: the encrypted path cannot serve (%s)", (runtime or Runtime.current()).openssl)
raise TlsUnavailable("tls13_required")
def start_decision(runtime: Runtime | None = None) -> dict[str, str]:
"""May the terminator start here, as the status file says it: ``ready`` or ``degraded`` with the closed word that says why."""
if tls13_available(runtime):
return {"state": "ready", "reason": ""}
return {"state": "degraded", "reason": "tls13_required"}
_VERSIONS = {"TLSv1.3": "TLSv1_3", "TLSv1.2": "TLSv1_2"}
def minimum_version() -> "ssl.TLSVersion":
word = policy.value("terminator.tls_min_version")
if word not in _VERSIONS:
raise TlsUnavailable("tls13_required")
return getattr(ssl.TLSVersion, _VERSIONS[word])
def server_context(chain_path: Path | str, key_path: Path | str, *, runtime: Runtime | None = None) -> ssl.SSLContext:
"""The terminator's context for one certificate generation: TLS 1.3 at least, ALPN of the policy, nothing narrower. Raises ``TlsUnavailable`` when the Python
has no TLS 1.3; every other property of the library (version, hybrid) is reported, not enforced."""
require_tls13(runtime)
context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
context.minimum_version = minimum_version()
context.set_alpn_protocols([policy.value("terminator.tls_alpn")])
context.load_cert_chain(str(chain_path), str(key_path))
return context
# ------------------------------------------------------------------------------------------------------------------------ OPENSSL_CONF
def render_openssl_conf() -> str:
"""The text of the installer's OpenSSL config file: groups and TLS 1.3 suites only. No ``MinProtocol`` (an outbound TLS 1.2 peer, a corporate proxy, must keep working),
and the hybrid group behind ``?`` so that an OpenSSL that does not know it ignores the name instead of refusing the whole list."""
return (
"# ALL 2.0: key-exchange groups and TLS 1.3 suites for the e2e child process. OFF unless runtime.env says ALL_E2E_OPENSSL_CONF=on; see src/all_mcp/tls_policy.py.\n"
"# Generated by tls_policy.render_openssl_conf(): a test compares this file with it.\n"
"openssl_conf = all_openssl_init\n\n"
"[all_openssl_init]\n"
"ssl_conf = all_ssl\n\n"
"[all_ssl]\n"
"system_default = all_tls_defaults\n\n"
"[all_tls_defaults]\n"
f"Groups = {OPENSSL_CONF_GROUPS}\n"
f"Ciphersuites = {OPENSSL_CONF_SUITES}\n"
)
def openssl_conf_wanted(env: Mapping[str, str]) -> bool:
word = str(env.get(OPENSSL_CONF_SWITCH, "")).strip().lower()
return word in ON_WORDS if word else OPENSSL_CONF_DEFAULT_ON
def apply_openssl_conf(env: Mapping[str, str], directory: Path | str | None = None, *, switch: str | None = None) -> dict[str, str]:
"""The environment of the e2e child: a copy of `env`, with ``OPENSSL_CONF`` pointing at the restricted configuration only when it was turned on (`switch`, a
word of runtime.env, or ``ALL_E2E_OPENSSL_CONF`` in `env`). Off (the default) returns the environment unchanged, so the child behaves as it did without this
module. A file that cannot be written leaves the environment as it was: a policy nice-to-have never stops the child."""
out = dict(env)
wanted = openssl_conf_wanted({OPENSSL_CONF_SWITCH: switch} if switch is not None else out)
if not wanted or directory is None:
return out
try:
target = Path(directory) / OPENSSL_CONF_FILE
target.parent.mkdir(parents=True, exist_ok=True)
text = render_openssl_conf()
if not target.exists() or target.read_text(encoding="utf-8") != text:
target.write_text(text, encoding="utf-8")
out["OPENSSL_CONF"] = str(target)
except OSError as exc:
log.warning("the OpenSSL restriction could not be written (%s): the child runs without it", type(exc).__name__)
return out
# ------------------------------------------------------------------------------------------------------------------------ loopback handshakes
@dataclass(frozen=True)
class ClientProfile:
"""One kind of client for a probe. ``groups`` and ``ciphersuites`` are OpenSSL lists applied to a child process (Python cannot restrict a client's groups
in-process); a profile without them runs in-process. ``max_version`` is a policy word (``TLSv1.2``)."""
name: str
groups: str | None = None
ciphersuites: str | None = None
max_version: str | None = None
alpn: tuple[str, ...] = ("http/1.1",)
server_name: str = PROBE_NAME
@property
def in_process(self) -> bool:
return self.groups is None and self.ciphersuites is None
HYBRID_CLIENT = ClientProfile("hybrid_client")
CLASSIC_CLIENT = ClientProfile("classic_client", groups="X25519")
TLS12_CLIENT = ClientProfile("tls12_client", max_version="TLSv1.2")
@dataclass
class Handshake:
"""The outcome of one loopback handshake: what each side sent in the clear, parsed, and what the two ends reported themselves."""
profile: str
completed: bool = False
error: str = ""
client_hello: ClientHelloInfo | None = None
server: ServerObservation = field(default_factory=ServerObservation)
python_version: str | None = None # the server's own view, to cross-check the parser against
python_cipher: str | None = None
alpn: str | None = None
client_error: str = ""
client_bytes: bytes = b"" # what each side wrote on the wire: a gate turns them into a capture, a test into a hello to cut and mutate
server_bytes: bytes = b""
@property
def negotiated(self) -> ServerHelloInfo | None:
return self.server.negotiated
def as_dict(self) -> dict[str, Any]:
hello = self.negotiated
out: dict[str, Any] = {"completed": self.completed, "error": self.error, "alpn": self.alpn,
"client": self.client_hello.as_dict() if self.client_hello else None,
"server": hello.as_dict() if hello and self.completed else None,
"alert": self.server.alert.name if self.server.alert else None}
if hello and self.completed:
out["consistent"] = (hello.version_name == self.python_version and hello.cipher_name == self.python_cipher)
return out
class _InProcessClient:
def __init__(self, profile: ClientProfile):
context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE # a loopback probe of our own server with a throw-away certificate: nothing is trusted, nothing is sent
if profile.max_version:
context.maximum_version = getattr(ssl.TLSVersion, _VERSIONS[profile.max_version])
if profile.alpn:
context.set_alpn_protocols(list(profile.alpn))
self._in, self._out = ssl.MemoryBIO(), ssl.MemoryBIO()
self._ssl = context.wrap_bio(self._in, self._out, server_hostname=profile.server_name)
self.error = ""
def step(self, data: bytes) -> tuple[bytes, bool]:
if data:
self._in.write(data)
done = False
try:
self._ssl.do_handshake()
done = True
except ssl.SSLWantReadError:
pass
except ssl.SSLError as exc:
self.error = (getattr(exc, "reason", None) or type(exc).__name__).lower()
done = True
return self._out.read(), done
def close(self) -> None:
return None
# The child of a probe: a client that reads the server's bytes from stdin and writes its own to stdout, one frame each (4-byte length, a tag, the bytes), so a probe
# needs no port and works inside any sandbox. Tags: D = bytes, a reply is expected; F = the last bytes (no reply), R = the result as JSON.
_CHILD = r"""
import json, ssl, struct, sys
cfg = json.loads(sys.argv[1])
def put(tag, payload):
sys.stdout.buffer.write(struct.pack(">I", len(payload) + 1) + tag + payload); sys.stdout.buffer.flush()
def get():
head = sys.stdin.buffer.read(4)
if len(head) < 4: raise SystemExit(0)
body = sys.stdin.buffer.read(struct.unpack(">I", head)[0])
return body[1:]
ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
if cfg["max_version"]:
ctx.maximum_version = getattr(ssl.TLSVersion, cfg["max_version"].replace("TLSv1.", "TLSv1_"))
if cfg["alpn"]:
ctx.set_alpn_protocols(cfg["alpn"])
i, o = ssl.MemoryBIO(), ssl.MemoryBIO()
s = ctx.wrap_bio(i, o, server_hostname=cfg["server_name"])
data = b""
while True:
if data:
i.write(data)
done, err = False, ""
try:
s.do_handshake(); done = True
except ssl.SSLWantReadError:
pass
except ssl.SSLError as exc:
done, err = True, (getattr(exc, "reason", None) or type(exc).__name__).lower()
out = o.read()
if done:
put(b"F", out)
put(b"R", json.dumps({"error": err}).encode())
break
put(b"D", out)
data = get()
"""
class _ChildClient:
"""A client in a child process whose OpenSSL is configured by a conf file made for the profile (groups, suites): the only way to make a client that does not
offer a group the library knows. The child is the same interpreter, so the library under test is the one the product ships."""
def __init__(self, profile: ClientProfile, workdir: Path):
lines = ["openssl_conf = probe_init", "", "[probe_init]", "ssl_conf = probe_ssl", "", "[probe_ssl]", "system_default = probe_defaults", "", "[probe_defaults]"]
if profile.groups:
lines.append(f"Groups = {profile.groups}")
if profile.ciphersuites:
lines.append(f"Ciphersuites = {profile.ciphersuites}")
conf = workdir / f"{profile.name}.cnf"
conf.write_text("\n".join(lines) + "\n", encoding="utf-8")
env = {key: value for key, value in os.environ.items() if key != "OPENSSL_CONF"}
env["OPENSSL_CONF"] = str(conf)
config = json.dumps({"max_version": profile.max_version, "alpn": list(profile.alpn), "server_name": profile.server_name})
self._proc = subprocess.Popen([sys.executable, "-I", "-c", _CHILD, config], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
env=env, creationflags=no_window_flags())
self._frames: "queue.Queue[tuple[bytes, bytes] | None]" = queue.Queue()
self._reader = threading.Thread(target=self._read, daemon=True)
self._reader.start()
self.error = ""
self._first = True
def _read(self) -> None:
try:
stdout = self._proc.stdout
while stdout is not None:
head = stdout.read(4)
if len(head) < 4:
break
body = stdout.read(struct.unpack(">I", head)[0])
self._frames.put((body[:1], body[1:]))
except (OSError, ValueError):
pass
self._frames.put(None)
def step(self, data: bytes) -> tuple[bytes, bool]:
if not self._first:
try:
assert self._proc.stdin is not None
self._proc.stdin.write(struct.pack(">I", len(data) + 1) + b"D" + data)
self._proc.stdin.flush()
except (OSError, ValueError):
return b"", True
self._first = False
try:
frame = self._frames.get(timeout=PROBE_SECONDS)
except queue.Empty:
self.error = "probe_timeout"
return b"", True
if frame is None:
self.error = "probe_child_failed"
return b"", True
tag, payload = frame
if tag == b"F":
tail = self._frames.get(timeout=PROBE_SECONDS)
if tail is not None and tail[0] == b"R":
self.error = str(json.loads(tail[1].decode("utf-8")).get("error", ""))
return payload, True
return payload, False
def close(self) -> None:
with contextlib.suppress(Exception):
if self._proc.stdin:
self._proc.stdin.close()
with contextlib.suppress(Exception):
self._proc.wait(timeout=5)
if self._proc.poll() is None:
self._proc.kill()
with contextlib.suppress(Exception):
self._proc.wait(timeout=5)
if self._proc.stdout:
with contextlib.suppress(Exception):
self._proc.stdout.close()
def handshake(context: ssl.SSLContext, profile: ClientProfile = HYBRID_CLIENT, *, workdir: Path | None = None) -> Handshake:
"""A real TLS handshake between `context` (a server context) and a client of `profile`, pumped through memory (no socket), with every byte each side wrote kept
and parsed: the ClientHello, and the ServerHello or the alert the server answered with. Never raises for a refused handshake: that is the answer."""
result = Handshake(profile=profile.name)
server_in, server_out = ssl.MemoryBIO(), ssl.MemoryBIO()
server = context.wrap_bio(server_in, server_out, server_side=True)
scratch = None
if profile.in_process:
client: Any = _InProcessClient(profile)
else:
if workdir is None:
scratch = tempfile.TemporaryDirectory(prefix="all-tls-probe-")
workdir = Path(scratch.name)
client = _ChildClient(profile, workdir)
sent_by_client = bytearray()
sent_by_server = bytearray()
server_done = False
try:
data = b""
for _ in range(PROBE_ROUNDS):
to_server, client_done = client.step(data)
sent_by_client += to_server
if to_server:
server_in.write(to_server)
if not server_done:
try:
server.do_handshake()
server_done = True
except ssl.SSLWantReadError:
pass
except ssl.SSLError as exc:
result.error = (getattr(exc, "reason", None) or type(exc).__name__).lower()
server_done = True
data = server_out.read()
sent_by_server += data
if server_done or (client_done and not to_server):
break
result.client_error = getattr(client, "error", "")
result.completed = server_done and not result.error and not result.client_error
if result.completed:
result.python_version = server.version()
cipher = server.cipher()
result.python_cipher = cipher[0] if cipher else None
result.alpn = server.selected_alpn_protocol()
finally:
client.close()
if scratch is not None:
scratch.cleanup()
result.client_bytes, result.server_bytes = bytes(sent_by_client), bytes(sent_by_server)
with contextlib.suppress(TlsParseError):
result.client_hello = parse_client_hello(bytes(sent_by_client))
with contextlib.suppress(TlsParseError):
result.server = observe_server(bytes(sent_by_server))
return result
def probe_certificate(directory: Path) -> tuple[Path, Path]:
"""A throw-away ECDSA P-256 certificate for ``probe.all.invalid`` (chain and key, 0600), good for a day: the loopback probe's own, never a device key."""
from cryptography import x509
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import ec
from cryptography.x509.oid import NameOID
key = ec.generate_private_key(ec.SECP256R1())
name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, PROBE_NAME)])
now = datetime.datetime.now(datetime.timezone.utc)
cert = (x509.CertificateBuilder().subject_name(name).issuer_name(name).public_key(key.public_key()).serial_number(x509.random_serial_number())
.not_valid_before(now - datetime.timedelta(minutes=5)).not_valid_after(now + datetime.timedelta(days=1))
.add_extension(x509.SubjectAlternativeName([x509.DNSName(PROBE_NAME)]), critical=False).sign(key, hashes.SHA256()))
chain, private = directory / "probe-chain.pem", directory / "probe-key.pem"
private.write_bytes(key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption()))
chain.write_bytes(cert.public_bytes(serialization.Encoding.PEM))
with contextlib.suppress(OSError):
os.chmod(private, 0o600)
return chain, private
# ------------------------------------------------------------------------------------------------------------------------ the report
def _claim(provable: bool, evidence: str = "", reason: str = "") -> dict[str, Any]:
return {"provable": provable, "evidence": evidence if provable else "", "reason": "" if provable else reason}
def evaluate_claims(runtime: Runtime, probes: Mapping[str, Handshake | None], minimum_ok: bool) -> dict[str, dict[str, Any]]:
"""Which claims the probes prove, each with its evidence or the closed word that says why not. The OpenSSL version gates the hybrid claim and no other."""
default, classic, old = probes.get("hybrid_client"), probes.get("classic_client"), probes.get("tls12_client")
claims: dict[str, dict[str, Any]] = {}
hello = default.negotiated if default and default.completed else None
if not runtime.has_tls13:
for name in CLAIMS:
claims[name] = _claim(False, reason="tls13_required")
return claims
claims["tls_1_3_minimum"] = (_claim(True, CLAIMS["tls_1_3_minimum"]) if minimum_ok and hello is not None and hello.version == TLS_13
else _claim(False, reason="probe_failed" if hello is None else "not_tls13"))
if old is None:
claims["tls_1_2_refused"] = _claim(False, reason="not_probed")
elif not old.completed and (old.server.alert is not None or old.error):
claims["tls_1_2_refused"] = _claim(True, CLAIMS["tls_1_2_refused"])
else:
claims["tls_1_2_refused"] = _claim(False, reason="tls12_client_was_served" if old.completed else "inconclusive")
claims["alpn_http_1_1"] = (_claim(True, CLAIMS["alpn_http_1_1"]) if default and default.completed and default.alpn == policy.value("terminator.tls_alpn")
else _claim(False, reason="probe_failed" if hello is None else "alpn_not_selected"))
claims["forward_secrecy"] = (_claim(True, CLAIMS["forward_secrecy"]) if hello is not None and hello.version == TLS_13 and hello.group is not None
else _claim(False, reason="probe_failed" if hello is None else "no_ephemeral_group"))
if classic is None:
claims["classical_fallback"] = _claim(False, reason="not_probed")
elif classic.client_hello is not None and classic.client_hello.offers_hybrid:
claims["classical_fallback"] = _claim(False, reason="openssl_conf_ignored") # the child still offered hybrid: this runtime does not honour OPENSSL_CONF
elif classic.completed and classic.negotiated is not None and not classic.negotiated.hybrid:
claims["classical_fallback"] = _claim(True, CLAIMS["classical_fallback"])
else:
claims["classical_fallback"] = _claim(False, reason="classic_client_refused")
if not runtime.hybrid_capable:
claims["hybrid_x25519mlkem768"] = _claim(False, reason="openssl_older_than_3_5")
elif default is None or default.client_hello is None or not default.client_hello.offers_hybrid:
claims["hybrid_x25519mlkem768"] = _claim(False, reason="client_did_not_offer_hybrid")
elif hello is not None and hello.group == GROUP_X25519_MLKEM768:
claims["hybrid_x25519mlkem768"] = _claim(True, CLAIMS["hybrid_x25519mlkem768"])
else:
claims["hybrid_x25519mlkem768"] = _claim(False, reason="group_not_negotiated")
return {name: claims[name] for name in CLAIMS}
def capability_report(*, probe: bool = True, chain_path: Path | str | None = None, key_path: Path | str | None = None, runtime: Runtime | None = None,
environ: Mapping[str, str] | None = None, now: float | None = None) -> dict[str, Any]:
"""The per-OS TLS report of this computer, as the JSON that feeds ``e2e-proof.json``: the runtime, the start decision, the loopback probes and the claims they
prove (``allowed_claims``). `runtime` can be replaced to describe another library (tests); the probes always run on the real one. With ``probe=False`` nothing is
handshaken and no claim is provable (the cheap form the status file may carry)."""
runtime = runtime or Runtime.current()
environ = os.environ if environ is None else environ
decision = start_decision(runtime)
report: dict[str, Any] = {"v": REPORT_VERSION, "kind": "tls_capability_report",
"generated_at": (datetime.datetime.now(datetime.timezone.utc) if now is None
else datetime.datetime.fromtimestamp(now, datetime.timezone.utc)).strftime("%Y-%m-%dT%H:%M:%SZ"),
**runtime.as_dict(),
"policy": {"min_version": policy.value("terminator.tls_min_version"), "alpn": policy.value("terminator.tls_alpn")},
"terminator": {"can_start": decision["state"] == "ready", **decision},
"openssl_conf": {"default_on": OPENSSL_CONF_DEFAULT_ON, "switch": OPENSSL_CONF_SWITCH, "in_environment": bool(environ.get("OPENSSL_CONF")),
"effective": None},
"probes": {}, "claims": {}, "allowed_claims": [], "never_claimed": dict(NEVER_CLAIMED)}
probes: dict[str, Handshake | None] = {}
if probe and runtime.has_tls13 and tls13_available():
with tempfile.TemporaryDirectory(prefix="all-tls-report-") as scratch:
folder = Path(scratch)
if chain_path is None or key_path is None:
chain_path, key_path = probe_certificate(folder)
context = server_context(chain_path, key_path)
minimum_ok = context.minimum_version >= ssl.TLSVersion.TLSv1_3
for profile in (HYBRID_CLIENT, CLASSIC_CLIENT, TLS12_CLIENT):
try:
probes[profile.name] = handshake(context, profile, workdir=folder)
except Exception as exc: # noqa: BLE001 - a probe that cannot run is reported as not proven, never a crash of the report
log.warning("TLS probe %s failed (%s)", profile.name, type(exc).__name__)
probes[profile.name] = None
report["probes"] = {name: (shake.as_dict() if shake else {"completed": False, "error": "probe_failed"}) for name, shake in probes.items()}
classic = probes.get("classic_client")
if classic is not None and classic.client_hello is not None:
if runtime.hybrid_capable: # a library that has no hybrid group shows nothing about whether it honoured the file
report["openssl_conf"]["effective"] = not classic.client_hello.offers_hybrid
report["claims"] = evaluate_claims(runtime, probes, minimum_ok)
else:
reason = "tls13_required" if not runtime.has_tls13 else "not_probed"
report["claims"] = {name: _claim(False, reason=reason) for name in CLAIMS}
report["allowed_claims"] = [name for name, claim in report["claims"].items() if claim["provable"]]
return report
def validate_report(report: Any) -> list[str]:
"""What is wrong with a capability report, as closed words (an empty list is a well-formed one): the check the gate runs before it copies a report into
``e2e-proof.json``, and the one the site's TLS sentences rely on."""
problems: list[str] = []
if not isinstance(report, dict):
return ["not_an_object"]
if report.get("v") != REPORT_VERSION or report.get("kind") != "tls_capability_report":
problems.append("wrong_kind_or_version")
for key in ("os", "python", "openssl", "policy", "terminator", "probes", "claims", "allowed_claims", "never_claimed", "generated_at"):
if key not in report:
problems.append("missing_" + key)
claims = report.get("claims") if isinstance(report.get("claims"), dict) else {}
if set(claims) != set(CLAIMS):
problems.append("claims_not_the_known_set")
allowed = report.get("allowed_claims")
if not isinstance(allowed, list) or allowed != [name for name in CLAIMS if claims.get(name, {}).get("provable") is True]:
problems.append("allowed_claims_do_not_match_claims")
openssl = report.get("openssl") if isinstance(report.get("openssl"), dict) else {}
if claims.get("hybrid_x25519mlkem768", {}).get("provable") and not openssl.get("hybrid_capable"):
problems.append("hybrid_claim_without_openssl_3_5")
if claims.get("tls_1_3_minimum", {}).get("provable") and not openssl.get("has_tls13"):
problems.append("tls13_claim_without_tls13")
for name, claim in claims.items():
if not isinstance(claim, dict) or not isinstance(claim.get("provable"), bool):
problems.append("claim_malformed")
elif claim["provable"] and not claim.get("evidence"):
problems.append("claim_without_evidence_" + name)
elif not claim["provable"] and not claim.get("reason"):
problems.append("claim_without_reason_" + name)
return problems
def proof_section(report: Mapping[str, Any]) -> dict[str, Any]:
"""The part of a report that ``e2e-proof.json`` carries for this operating system: who it is, which library, what the loopback showed and the claims it
allows. The blindness gate adds the ServerHello counts of the real capture next to it (``summarize``)."""
observed = (report.get("probes", {}).get("hybrid_client") or {}).get("server")
return {"os": report.get("os"), "python": report.get("python"), "openssl": report.get("openssl"), "terminator": report.get("terminator"),
"loopback": observed, "allowed_claims": list(report.get("allowed_claims", [])), "generated_at": report.get("generated_at")}
def main(argv: Sequence[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("command", choices=["report"])
parser.add_argument("--out", type=Path, help="write the JSON here as well")
parser.add_argument("--no-probe", action="store_true", help="runtime facts only: no handshake, no claim proven")
args = parser.parse_args(argv)
report = capability_report(probe=not args.no_probe)
text = json.dumps(report, indent=2, sort_keys=True)
if args.out:
args.out.write_text(text + "\n", encoding="utf-8")
print(text)
return 0 if report["terminator"]["can_start"] else 3
if __name__ == "__main__":
hide_child_consoles() # Windows: the probe's child interpreter opens no console window
sys.exit(main())
files/src/all_mcp/tls_store.py 660 lines · 29.9 KB
"""ALL 2.0: the device's TLS key and certificate chain on disk, as GENERATIONS (plan 2.0, sections 2 and 4).
Every issuance makes a NEW EC P-256 key and writes it, together with the chain the CA returns for it, into its own folder ``tls/gen-<n>/``.
Which generation serves is a tiny file, ``tls/current``, that is only ever replaced by an atomic rename. So:
* the key that is serving is never overwritten, and the pointer never names a generation whose chain is not on disk and matches its key;
* a crash at any moment leaves the previous pointer (and so the previous certificate) in force, plus at worst a folder that nothing names;
* the previous generation is kept until the new one has served a handshake (``mark_served``), so a certificate that turns out not to work can
still be gone back to;
* nothing here follows a link: every file goes through ``_device_storage.PrivateStore`` (no symbolic links, one hard link, this user's, 0600,
in a 0700 folder; an owner-only DACL on Windows), exactly like the device identity.
Layout of ``tls/`` (the folder is 0700, every file 0600):
current {"v":1,"gen":n,"previous":m|null,"served":bool} the pointer
gen-<n>/ device.key, fullchain.pem, meta.json (host, CA, validity, SPKI hash)
state.json what the certificate manager remembers between runs (attempts, back-off, ARI, a confirmation still to send)
status.json what the tray, the runner and the fleet heartbeat read (no secret, never a TXT value)
spki-log append-only, one JSON line per key this device ever made: the anchor of the device's own CT self-watch (WP36)
ct-watch.json what the CT self-watch remembers: its alerts, their acknowledgements, the last look at CT (e2e_ct_watch.py; ct-watch.lock is its lock)
key-storage.json how the keys are kept (a file today) and whether a native store could hold them; it never changes by itself
acme-account.key, acme-account.json, manager.lock
Constants come from contracts/policy.json (the copy in e2e_policy.json), never from here.
"""
from __future__ import annotations
import base64
import datetime
import errno
import functools
import hashlib
import json
import logging
import os
import re
import secrets
import shutil
import ssl
import threading
import time
from collections.abc import Callable, Iterator
from contextlib import contextmanager
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from cryptography import x509
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import ec
from . import tls_policy
from ._device_storage import CredentialStorageError, LockTimeout, PrivateStore, held_lock
from .config import state_dir
log = logging.getLogger("all_mcp.tls_store")
POINTER = "current"
STATE = "state.json"
STATUS = "status.json"
SPKI_LOG = "spki-log"
KEY_FILE = "device.key"
CHAIN_FILE = "fullchain.pem"
META_FILE = "meta.json"
KEY_STORAGE = "key-storage.json"
LOCK_FILE = "manager.lock"
GENERATION_RE = re.compile(r"gen-([1-9][0-9]{0,8})")
TEMPORARY_PREFIX = ".tmp-"
SCHEMA_VERSION = 1
MAX_POINTER_BYTES = 4096
UNSAFE_ERRNOS = {errno.ELOOP, errno.ENOTDIR, errno.EISDIR, errno.EPERM, errno.EACCES}
class TlsStoreError(Exception):
"""``word`` is a closed word (never text from the network) the status file and the heartbeat may carry."""
def __init__(self, word: str, message: str = ""):
super().__init__(message or word)
self.word = word
@functools.lru_cache(maxsize=1)
def policy() -> dict[str, Any]:
"""The device's copy of contracts/policy.json (the contract generator sync keeps it byte-identical)."""
return json.loads(Path(__file__).with_name("e2e_policy.json").read_text(encoding="utf-8"))
def certificate_policy() -> dict[str, Any]:
return policy()["certificates"]
def b64url(data: bytes) -> str:
return base64.urlsafe_b64encode(data).rstrip(b"=").decode("ascii")
# ---------------------------------------------------------------------------------------------------------------- reading a chain
@dataclass(frozen=True)
class CertInfo:
names: tuple[str, ...]
not_before: float
not_after: float
fingerprint: str # SHA-256 of the leaf, hex
issuer: str
@property
def renew_at(self) -> float:
"""When only the share of the life that policy.certificates.renew_backstop_divisor names is left (a third: day 30 of 45, day 60 of 90)."""
return self.not_after - (self.not_after - self.not_before) / certificate_policy()["renew_backstop_divisor"]
def _utc(certificate: x509.Certificate, attribute: str) -> float:
value = getattr(certificate, attribute + "_utc", None)
if value is None: # cryptography < 42
value = getattr(certificate, attribute).replace(tzinfo=datetime.timezone.utc)
return value.timestamp()
def parse_chain(pem: bytes) -> list[x509.Certificate]:
certificates = x509.load_pem_x509_certificates(pem)
if not certificates:
raise ValueError("no certificate in the chain")
return certificates
def inspect_chain(pem: bytes) -> CertInfo:
leaf = parse_chain(pem)[0]
try:
names = tuple(leaf.extensions.get_extension_for_class(x509.SubjectAlternativeName).value.get_values_for_type(x509.DNSName))
except x509.ExtensionNotFound:
names = ()
return CertInfo(names=tuple(n.lower() for n in names), not_before=_utc(leaf, "not_valid_before"), not_after=_utc(leaf, "not_valid_after"),
fingerprint=leaf.fingerprint(hashes.SHA256()).hex(), issuer=leaf.issuer.rfc4514_string())
def _public_der(public_key) -> bytes:
return public_key.public_bytes(serialization.Encoding.DER, serialization.PublicFormat.SubjectPublicKeyInfo)
def chain_matches_key(pem: bytes, key) -> bool:
return _public_der(parse_chain(pem)[0].public_key()) == _public_der(key.public_key())
def spki_sha256(public_key) -> str:
"""base64url (43 characters) of the SHA-256 of the SubjectPublicKeyInfo: what the broker records before any order and what CT is compared with."""
return b64url(hashlib.sha256(_public_der(public_key)).digest())
def key_pem(key) -> bytes:
return key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption())
def new_key() -> ec.EllipticCurvePrivateKey:
return ec.generate_private_key(ec.SECP256R1())
def leaf_der(pem: bytes) -> bytes:
return parse_chain(pem)[0].public_bytes(serialization.Encoding.DER)
# ---------------------------------------------------------------------------------------------------------------- one generation
@dataclass(frozen=True)
class Generation:
number: int
key_path: Path
chain_path: Path
info: CertInfo
spki: str
meta: dict[str, Any]
chain_pem: bytes = field(default=b"", repr=False)
@property
def ca(self) -> str:
return str(self.meta.get("ca") or "")
@property
def renew_at(self) -> float:
return self.info.renew_at
def _pointer_value(document: Any) -> dict[str, Any]:
if (not isinstance(document, dict) or document.get("v") != SCHEMA_VERSION or type(document.get("gen")) is not int or document["gen"] < 1
or not (document.get("previous") is None or (type(document["previous"]) is int and document["previous"] >= 1))
or not isinstance(document.get("served"), bool)):
raise TlsStoreError("pointer_invalid")
return {"v": SCHEMA_VERSION, "gen": document["gen"], "previous": document.get("previous"), "served": document["served"]}
class TlsStore:
"""Everything under ``tls/``. ``faults`` is a hook for the tests (and only them): it is called with the name of a step of an issuance
("after_key", "mid_chain", "before_swap", "after_swap") and may raise to stop the process exactly there."""
def __init__(self, directory: Path | None = None, *, faults: Callable[[str], None] | None = None):
self.dir = Path(directory) if directory is not None else state_dir() / "tls"
self.faults = faults
def hit(self, point: str) -> None:
if self.faults is not None:
self.faults(point)
# -- access
@contextmanager
def _open(self, *, create: bool = True) -> Iterator[PrivateStore | None]:
"""The folder, opened through ``PrivateStore`` (so a link, a stranger's folder or a wrong mode refuses). Unsafe storage is
``TlsStoreError("storage_unsafe")``; with ``create=False`` a folder that does not exist yields None."""
try:
with self._guard():
store = PrivateStore(self.dir, create=create)
except FileNotFoundError:
yield None
return
try:
with self._guard():
yield store
finally:
store.close()
@staticmethod
@contextmanager
def _guard() -> Iterator[None]:
"""What the storage checks refuse (a link, a stranger's file, a wrong mode, a second hard link) is ``storage_unsafe``; any other failure of the
disk (full, read-only, an I/O error) is ``storage_error``: not a reason to distrust the folder, and not something to repair by deleting."""
try:
yield
except (TlsStoreError, LockTimeout, FileNotFoundError):
raise
except CredentialStorageError:
raise TlsStoreError("storage_unsafe") from None
except OSError as exc:
raise TlsStoreError("storage_unsafe" if exc.errno in UNSAFE_ERRNOS else "storage_error") from None
@contextmanager
def exclusive(self, timeout: float = 0.0) -> Iterator[None]:
"""One certificate manager at a time, across processes (a second pass that finds it taken gets ``LockTimeout`` at once)."""
with self._open() as store, held_lock(store, LOCK_FILE, timeout):
yield
@contextmanager
def locked(self, name: str, timeout: float = 5.0) -> Iterator[None]:
"""A lock of its own on the private file ``name`` for a reader-modifier of one small record (the CT self-watch's state, e2e_ct_watch.py). It is not
``exclusive``: that one is held for a whole issuance, and a record the tray rewrites must not wait for it."""
with self._open() as store, held_lock(store, name, timeout):
yield
def _atomic(self, store: PrivateStore, name: str, data: bytes, *, fault: str | None = None) -> None:
"""Write ``data`` to ``name`` so that a reader sees the old file or the whole new one: a private temporary file (created 0600, so there is
no moment with wider permissions), fsync, atomic rename, fsync of the folder. With ``fault`` the step of that name fires half way through the
file, which is where a power loss leaves a half-written temporary and nothing else."""
temporary = f"{TEMPORARY_PREFIX}{secrets.token_hex(8)}"
descriptor = store.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL)
try:
with os.fdopen(descriptor, "wb") as stream:
half = len(data) // 2 if fault else len(data)
stream.write(data[:half])
stream.flush()
if fault:
self.hit(fault)
stream.write(data[half:])
stream.flush()
os.fsync(stream.fileno())
store.rename(temporary, name)
store.sync()
except Exception: # not BaseException: a simulated crash leaves the temporary behind, like the real thing (gc removes it)
store.remove(temporary)
raise
@staticmethod
def _json(store: PrivateStore, name: str) -> Any | None:
raw = store.read(name)
if raw is None:
return None
try:
return json.loads(raw.decode("utf-8"))
except (ValueError, UnicodeError):
raise TlsStoreError("file_invalid", name) from None
# -- the pointer
def pointer(self) -> dict[str, Any] | None:
with self._open(create=False) as store:
return None if store is None else self._pointer(store)
def _pointer(self, store: PrivateStore) -> dict[str, Any] | None:
raw = store.read(POINTER)
if raw is None:
return None
if len(raw) > MAX_POINTER_BYTES:
raise TlsStoreError("pointer_invalid")
try:
return _pointer_value(json.loads(raw.decode("utf-8")))
except (ValueError, UnicodeError):
raise TlsStoreError("pointer_invalid") from None
def _numbers(self, store: PrivateStore) -> list[int]:
return sorted(int(m.group(1)) for name in store.names() if (m := GENERATION_RE.fullmatch(name)) and store.kind(name) == "dir")
# -- generations
def new_generation(self) -> tuple[int, ec.EllipticCurvePrivateKey]:
"""A new key, written 0600 into a folder of its own (never over a live key), and its SPKI hash added to ``spki-log``. Nothing is
served from it until ``write_chain`` and ``activate``."""
key = new_key()
with self._open() as store:
pointer = None
try:
pointer = self._pointer(store)
except TlsStoreError:
pass # a pointer that cannot be read does not decide the number: the folders do
named = [pointer["gen"], pointer["previous"] or 0] if pointer else []
number = max([*self._numbers(store), *named], default=0) + 1
folder = store.child(f"gen-{number}", create=True)
try:
self._atomic(folder, KEY_FILE, key_pem(key))
finally:
folder.close()
line = json.dumps({"v": SCHEMA_VERSION, "spki_sha256": spki_sha256(key.public_key()), "gen": number, "at": int(time.time())},
separators=(",", ":")) + "\n"
store.append(SPKI_LOG, line.encode("ascii"))
self.hit("after_key")
return number, key
def write_chain(self, number: int, chain_pem: bytes, host: str, meta: dict[str, Any]) -> Generation:
"""Put the CA's chain beside the key it was issued for, and read the pair back from disk to validate it (the chain's leaf carries THIS key,
covers ``host``). A pair that does not match raises ``pair_mismatch`` and leaves nothing that could be activated."""
with self._open() as store:
folder = store.child(f"gen-{number}")
try:
key = self._read_key(folder)
self._check_pair(chain_pem, key, host)
self._atomic(folder, CHAIN_FILE, chain_pem, fault="mid_chain")
document = {"v": SCHEMA_VERSION, "host": host, **meta, "spki_sha256": spki_sha256(key.public_key())}
self._atomic(folder, META_FILE, json.dumps(document, sort_keys=True, separators=(",", ":")).encode("utf-8"))
finally:
folder.close()
return self._load(store, number)
@staticmethod
def _read_key(folder: PrivateStore) -> ec.EllipticCurvePrivateKey:
raw = folder.read(KEY_FILE)
if raw is None:
raise TlsStoreError("incomplete")
try:
key = serialization.load_pem_private_key(raw, password=None)
except (ValueError, TypeError):
raise TlsStoreError("key_invalid") from None
if not isinstance(key, ec.EllipticCurvePrivateKey) or not isinstance(key.curve, ec.SECP256R1):
raise TlsStoreError("key_invalid")
return key
@staticmethod
def _check_pair(chain_pem: bytes, key, host: str) -> CertInfo:
try:
info = inspect_chain(chain_pem)
if not chain_matches_key(chain_pem, key):
raise TlsStoreError("pair_mismatch")
except ValueError:
raise TlsStoreError("chain_invalid") from None
if host and host.lower() not in info.names:
raise TlsStoreError("pair_mismatch", "the certificate does not cover this device's name")
return info
def _load(self, store: PrivateStore, number: int) -> Generation:
try:
folder = store.child(f"gen-{number}")
except FileNotFoundError:
raise TlsStoreError("incomplete") from None
try:
key = self._read_key(folder)
chain = folder.read(CHAIN_FILE)
if chain is None:
raise TlsStoreError("incomplete")
meta = self._json(folder, META_FILE)
finally:
folder.close()
meta = meta if isinstance(meta, dict) else {}
info = self._check_pair(chain, key, str(meta.get("host") or ""))
return Generation(number, self.dir / f"gen-{number}" / KEY_FILE, self.dir / f"gen-{number}" / CHAIN_FILE, info, spki_sha256(key.public_key()), meta, chain)
def generation(self, number: int) -> Generation | None:
with self._open(create=False) as store:
if store is None:
return None
try:
return self._load(store, number)
except TlsStoreError:
return None
def generations(self) -> list[int]:
with self._open(create=False) as store:
return [] if store is None else self._numbers(store)
def active(self) -> Generation | None:
"""The generation to serve: the one the pointer names; when that one cannot be read as a valid pair, the previous one; else None."""
with self._open(create=False) as store:
if store is None:
return None
try:
pointer = self._pointer(store)
except TlsStoreError:
return None
if pointer is None:
return None
for number in (pointer["gen"], pointer["previous"]):
if number is None:
continue
try:
return self._load(store, number)
except TlsStoreError as exc:
log.warning("generation %s is not usable (%s)", number, exc.word)
return None
def activate(self, number: int) -> Generation:
"""Make ``number`` the serving generation: validate the pair again from disk, then replace the pointer by an atomic rename. The generation that
was serving becomes ``previous`` and stays until ``mark_served``."""
with self._open() as store:
generation = self._load(store, number)
try:
old = self._pointer(store)
except TlsStoreError:
old = None
previous = old["gen"] if old and old["gen"] != number else (old["previous"] if old else None)
self._atomic(store, POINTER, json.dumps({"v": SCHEMA_VERSION, "gen": number, "previous": previous, "served": False}, separators=(",", ":")).encode())
return generation
def mark_served(self, number: int) -> bool:
"""The terminator completed a handshake with generation ``number``: now the previous one can go. False when ``number`` is not the serving one."""
with self._open(create=False) as store:
if store is None:
return False
pointer = self._pointer(store)
if pointer is None or pointer["gen"] != number:
return False
if pointer["served"] and pointer["previous"] is None:
return True
self._atomic(store, POINTER, json.dumps({"v": SCHEMA_VERSION, "gen": number, "previous": None, "served": True}, separators=(",", ":")).encode())
if pointer["previous"] is not None:
store.remove_tree(f"gen-{pointer['previous']}")
return True
def discard(self, number: int) -> None:
"""Remove a generation that was never activated (a failed issuance). The serving and the previous one are refused."""
with self._open() as store:
try:
pointer = self._pointer(store)
except TlsStoreError:
pointer = None
if pointer and number in (pointer["gen"], pointer["previous"]):
raise TlsStoreError("in_use")
store.remove_tree(f"gen-{number}")
def resumable(self, host: str) -> list[Generation]:
"""Complete, valid generations that nothing names, newest first: what a crash between "chain written" and "pointer swapped" leaves behind.
Adopting one saves a whole issuance (the CA already counted it)."""
found: list[Generation] = []
with self._open(create=False) as store:
if store is None:
return found
try:
pointer = self._pointer(store)
except TlsStoreError:
pointer = None
named = {pointer["gen"], pointer["previous"]} if pointer else set()
for number in reversed(self._numbers(store)):
if number in named:
continue
try:
generation = self._load(store, number)
except TlsStoreError:
continue
if str(generation.meta.get("host") or "") == host and generation.meta.get("spki_sha256") == generation.spki:
found.append(generation)
return found
def gc(self) -> list[str]:
"""Remove what nothing names: generations that are neither the serving nor the previous one, and temporary files a crash left. Returns the
names removed. A folder that cannot be opened safely raises ``storage_unsafe`` and is left alone."""
removed: list[str] = []
with self._open() as store:
try:
pointer = self._pointer(store)
known = True
except TlsStoreError:
pointer, known = None, False # a pointer that cannot be read names nothing: so nothing may be thrown away because of it
keep = {f"gen-{pointer['gen']}", f"gen-{pointer['previous']}"} if pointer else set()
for name in store.names():
if name.startswith(TEMPORARY_PREFIX) or (known and GENERATION_RE.fullmatch(name) and name not in keep):
store.remove_tree(name)
removed.append(name)
elif GENERATION_RE.fullmatch(name) and known:
folder = store.child(name)
try:
for entry in folder.names():
if entry.startswith(TEMPORARY_PREFIX):
folder.remove(entry)
removed.append(f"{name}/{entry}")
finally:
folder.close()
return removed
def destroy(self) -> None:
"""Uninstall: everything under ``tls/`` goes, at once, with no network. The folder itself is removed too."""
try:
with self._open(create=False) as store:
if store is None:
return
for name in store.names():
store.remove_tree(name)
except TlsStoreError:
# a folder the checks refuse (a stranger's, a link): a link is only unlinked, never followed
if self.dir.is_symlink():
self.dir.unlink()
else:
shutil.rmtree(self.dir, ignore_errors=True)
return
try:
self.dir.rmdir()
except OSError:
pass
# -- records of the manager
def read_state(self) -> dict[str, Any] | None:
with self._open(create=False) as store:
if store is None:
return None
document = self._json(store, STATE)
return document if isinstance(document, dict) else None
def write_state(self, state: dict[str, Any]) -> None:
with self._open() as store:
self._atomic(store, STATE, json.dumps(state, sort_keys=True, separators=(",", ":")).encode("utf-8"))
def write_status(self, status: dict[str, Any]) -> None:
with self._open() as store:
self._atomic(store, STATUS, json.dumps(status, sort_keys=True, separators=(",", ":")).encode("utf-8"))
def read_status(self) -> dict[str, Any] | None:
with self._open(create=False) as store:
if store is None:
return None
document = self._json(store, STATUS)
return document if isinstance(document, dict) else None
def read_file(self, name: str) -> bytes | None:
with self._open(create=False) as store:
return None if store is None else store.read(name)
def write_file(self, name: str, data: bytes) -> None:
with self._open() as store:
self._atomic(store, name, data)
def spki_log(self) -> list[dict[str, Any]]:
"""Every key this device made, oldest first (what WP36 compares certificates seen in CT with)."""
with self._open(create=False) as store:
raw = None if store is None else store.read(SPKI_LOG)
rows = []
for line in (raw or b"").decode("ascii", "replace").splitlines():
try:
row = json.loads(line)
except ValueError:
continue
if isinstance(row, dict) and isinstance(row.get("spki_sha256"), str):
rows.append(row)
return rows
# -- how the keys are kept
def key_storage(self) -> dict[str, Any]:
"""The descriptor of key storage. Keys are files (0600) today; ``native_eligible`` says which native store (macOS Keychain, Windows
credential vault, Secret Service) COULD hold a wrapping key, so a tray or an installer can OFFER it. The mode never changes by itself: not when a
native store appears, not when one goes away. A descriptor that says anything but "file" is refused (``key_storage_unsupported``) instead of
reading the files as if nothing was said."""
with self._open() as store:
document = self._json(store, KEY_STORAGE)
if document is None:
document = {"v": SCHEMA_VERSION, "mode": "file", "wrapped_by": None, "change": "explicit"}
self._atomic(store, KEY_STORAGE, json.dumps(document, sort_keys=True, separators=(",", ":")).encode())
if not isinstance(document, dict) or document.get("mode") != "file" or document.get("wrapped_by") is not None:
raise TlsStoreError("key_storage_unsupported")
return {**document, "native_eligible": native_store_name()}
def native_store_name() -> str | None:
"""The native secret store this system could use to wrap the keys, or None. Looked up each time; nothing is switched by finding one."""
try:
from .device_credentials import _native_backend
found = _native_backend()
except Exception:
return None
return found[0] if found else None
# ---------------------------------------------------------------------------------------------------------------- serving
def server_context(chain_path: Path, key_path: Path) -> ssl.SSLContext:
"""The terminator's context for one generation. The policy (minimum version, ALPN, the refusal to serve without TLS 1.3) is tls_policy's."""
return tls_policy.server_context(chain_path, key_path)
class ServingContext:
"""The TLS context a terminator serves with, following the pointer. A new generation is picked up at the NEXT handshake: connections that are
already established keep the context (the certificate) they began with, so a swap drops no stream. A generation that cannot be loaded never
replaces the one in use.
serving = ServingContext(TlsStore())
listener = asyncio.start_server(handler, ssl=serving.listener_context()) # new handshakes follow the pointer by themselves
... after a handshake completed: serving.served()
"""
def __init__(self, store: TlsStore, *, build: Callable[[Path, Path], ssl.SSLContext] = server_context):
self.store = store
self._build = build
self._lock = threading.Lock()
self._number: int | None = None
self._context: ssl.SSLContext | None = None
self.last_error = ""
@property
def number(self) -> int | None:
return self._number
def current(self) -> ssl.SSLContext | None:
"""The context of the serving generation (loaded again when the pointer moved); the last good one when the pointer cannot be used; None
when there never was one."""
with self._lock:
try:
pointer = self.store.pointer()
except TlsStoreError as exc:
self.last_error = exc.word
return self._context
if pointer is not None and pointer["gen"] == self._number and self._context is not None:
return self._context
try:
generation = self.store.active()
if generation is None:
return self._context
if generation.number != self._number:
self._context = self._build(generation.chain_path, generation.key_path)
self._number = generation.number
self.last_error = ""
except (ssl.SSLError, OSError, TlsStoreError, tls_policy.TlsUnavailable, ValueError) as exc:
self.last_error = getattr(exc, "word", type(exc).__name__)
log.warning("the new certificate could not be loaded (%s): the one in use stays", self.last_error)
return self._context
def listener_context(self) -> ssl.SSLContext:
"""A context for ONE listener that follows the pointer: every handshake is moved onto the context of the generation serving at that moment."""
generation = self.store.active()
if generation is None:
raise TlsStoreError("no_generation")
listener = self._build(generation.chain_path, generation.key_path) # what answers until the callback moves the connection
def follow(connection, _name, _context) -> None:
fresh = self.current()
if fresh is not None and fresh is not connection.context:
connection.context = fresh
listener.sni_callback = follow
return listener
def served(self) -> bool:
"""A handshake completed with the generation in use: the previous generation can go."""
number = self._number
return False if number is None else self.store.mark_served(number)
files/src/all_mcp/tray/encryption_verify.py 143 lines · 9.1 KB
"""ALL 2.0 (WP37): run-all verify-source. Listed installed files must match the MANIFEST's SHA-256; nothing else is checked. Nothing is sent.
Which manifest decides what a pass means. The one bundled in the install lives beside the files it lists: whoever can change an installed file can change that manifest, so a pass
against it proves only that the two agree, never that the files are the ones ALL published. The command says exactly that, prints the manifest's own SHA-256 and names the way to
compare with the published one (a manifest downloaded from the page, passed with --manifest, whose SHA-256 the page also shows). Only that second pass says "the published ones"."""
from __future__ import annotations
import glob
import hashlib
import json
from pathlib import Path
from typing import Any
PACKAGE_DIR = Path(__file__).resolve().parent.parent # the all_mcp package this module is installed in
PREFIX = "src/all_mcp/"
BUNDLED_MANIFEST = PACKAGE_DIR / "encryption-source-manifest.json"
SCHEMA = "all-encryption-source/1"
EXIT_OK, EXIT_FAILED, EXIT_USAGE = 0, 1, 64
_TEXT = {
"es": {
"title": "Código publicado de ALL {version}: lo instalado",
"version_bad": "Instalada: {installed}. El manifiesto es de la versión {version}; usa el de tu versión.",
"no_manifest": "Esta copia no trae manifiesto. Usa: run-all verify-source --manifest RUTA (el de tu versión).",
"bad_manifest": "No pude leer el manifiesto: no es uno de ALL válido o está dañado.",
"changed": "Cambió: {path}",
"missing": "Falta: {path}",
"extra": "No está en el manifiesto: {path}",
"manifest": "Manifiesto: {source} (SHA-256 {digest})",
"source_bundled": "el que vino con esta instalación",
"source_file": "el archivo que indicaste",
"ok": "Correcto: los {count} archivos del manifiesto que corren en esta computadora coinciden con el manifiesto que indicaste. Compara el SHA-256 de arriba con el que muestra la página del código; si es el mismo, esos archivos son los publicados.",
"ok_bundled": "Los {count} archivos del manifiesto que corren en esta computadora coinciden con el manifiesto que vino con esta instalación. Eso solo muestra que ambos concuerdan: quien pudiera cambiar un archivo podría cambiar también ese manifiesto. Para compararlos con lo publicado, baja el manifiesto de tu versión de la página del código y corre: run-all verify-source --manifest RUTA",
"not_here": "{count} archivos del conjunto no corren en esta computadora. Este comando no los revisa.",
"failed": "No coincide: {count} problema(s) en los archivos del manifiesto.",
"usage": "uso: run-all verify-source [--manifest RUTA] [--json]",
},
"en": {
"title": "Published code of ALL {version}: what is installed",
"version_bad": "Installed: {installed}. The manifest is for {version}; use the one of your version.",
"no_manifest": "This copy has no manifest. Use: run-all verify-source --manifest PATH (the one of your version).",
"bad_manifest": "I could not read the manifest: it is not a valid ALL one or is damaged.",
"changed": "Changed: {path}",
"missing": "Missing: {path}",
"extra": "Not in the manifest: {path}",
"manifest": "Manifest: {source} (SHA-256 {digest})",
"source_bundled": "the one that came with this install",
"source_file": "the file you gave",
"ok": "OK: the {count} files of the manifest that run on this computer match the manifest you gave. Compare the SHA-256 above with the one the code page shows; if it is the same, those files are the published ones.",
"ok_bundled": "The {count} files of the manifest that run on this computer match the manifest that came with this install. That only shows the two agree: whoever could change a file could change that manifest too. To compare with what was published, download your version's manifest from the code page and run: run-all verify-source --manifest PATH",
"not_here": "{count} files of the set do not run on this computer. This command does not check them.",
"failed": "Does not match: {count} problem(s) in the files of the manifest.",
"usage": "usage: run-all verify-source [--manifest PATH] [--json]",
},
}
class ManifestError(Exception):
"""The _TEXT key that says why the manifest cannot be used."""
def _text(key: str, lang: str, **fields: Any) -> str:
return _TEXT["en" if lang == "en" else "es"][key].format(**fields)
def manifest_sha256(path: Path | None = None) -> str:
"""The SHA-256 of the manifest file itself (the bundled one when `path` is None): the number the code page shows beside the manifest it publishes."""
return hashlib.sha256((Path(path) if path is not None else BUNDLED_MANIFEST).read_bytes()).hexdigest()
def load_manifest(path: Path | None = None) -> dict[str, Any]:
source = Path(path) if path is not None else BUNDLED_MANIFEST
if not source.is_file():
raise ManifestError("no_manifest")
try:
manifest = json.loads(source.read_text(encoding="utf-8"))
if (manifest.get("schema") == SCHEMA and isinstance(manifest.get("allowlist"), list)
and all(isinstance(e.get("path"), str) and isinstance(e.get("sha256"), str) for e in manifest["files"])):
return manifest
except (OSError, ValueError, AttributeError, TypeError, KeyError):
pass
raise ManifestError("bad_manifest")
def verify(manifest: dict[str, Any], *, installed_version: str, package_dir: Path = PACKAGE_DIR) -> dict[str, Any]:
"""`ok` only when the version matches, listed files run here and match, no allow-listed file is unlisted, and one file was checked."""
listed = {entry["path"] for entry in manifest["files"]}
out: dict[str, Any] = {"version": manifest.get("version"), "version_match": manifest.get("version") == installed_version,
"checked": [], "changed": [], "missing": [], "extra": [], "not_installed_here": []}
for entry in manifest["files"]:
rel = entry["path"]
local = package_dir / rel[len(PREFIX):]
if not rel.startswith(PREFIX):
out["not_installed_here"].append(rel)
elif not local.is_file():
out["missing"].append(rel)
else:
out["checked"].append(rel)
if hashlib.sha256(local.read_bytes()).hexdigest() != entry["sha256"]:
out["changed"].append(rel)
root = package_dir.resolve()
withheld = {rel for rel in manifest.get("withheld", []) if isinstance(rel, str)} # what the allow-list names and the export does not publish (the denylist): not an unlisted file
for pattern in (p for p in manifest["allowlist"] if p.startswith(PREFIX)):
for found in glob.glob(str(root / pattern[len(PREFIX):]), recursive=True, include_hidden=True):
rel = PREFIX + Path(found).relative_to(root).as_posix()
if Path(found).is_file() and rel not in listed and rel not in withheld and rel not in out["extra"]:
out["extra"].append(rel)
out["ok"] = bool(out["version_match"] and out["checked"] and not (out["changed"] or out["missing"] or out["extra"]))
return out
def run(args: list[str], lang: str, *, installed_version: str | None = None,
package_dir: Path = PACKAGE_DIR) -> tuple[int, list[str], dict[str, Any] | None]:
"""The command: (exit code, lines for the person's terminal, the report for --json)."""
if installed_version is None:
from .. import __version__ as installed_version # noqa: N813 (the version this build runs)
rest = [a for a in args if a != "--json"]
if rest and not (len(rest) == 2 and rest[0] == "--manifest"):
return EXIT_USAGE, [_text("usage", lang)], None
given = Path(rest[1]).expanduser() if rest else None
try:
manifest = load_manifest(given)
digest = manifest_sha256(given)
except ManifestError as error:
return EXIT_FAILED, [_text(str(error), lang)], None
except OSError:
return EXIT_FAILED, [_text("bad_manifest", lang)], None
report = verify(manifest, installed_version=installed_version, package_dir=package_dir)
report["manifest_source"] = "bundled" if given is None else "file"
report["manifest_sha256"] = digest
lines = [_text("title", lang, version=installed_version),
_text("manifest", lang, source=_text("source_bundled" if given is None else "source_file", lang), digest=digest)]
if not report["version_match"]:
lines.append(_text("version_bad", lang, installed=installed_version, version=report["version"]))
for key in ("changed", "missing", "extra"):
lines += [_text(key, lang, path=p) for p in report[key]]
if report["not_installed_here"]:
lines.append(_text("not_here", lang, count=len(report["not_installed_here"])))
as_json = report if "--json" in args else None
if report["ok"]:
return EXIT_OK, lines + [_text("ok_bundled" if given is None else "ok", lang, count=len(report["checked"]))], as_json
problems = len(report["changed"]) + len(report["missing"]) + len(report["extra"]) + (0 if report["version_match"] else 1)
return EXIT_FAILED, lines + [_text("failed", lang, count=max(problems, 1))], as_json
files/tests/_pcap_builder.py 118 lines · 5.2 KB
"""Builds classic pcap files for the pcap test suite and the capture test suite: Ethernet, Linux cooked v2 and raw-IP frames, IPv4 and
IPv6, TCP conversations whose segments can be split anywhere (a marker split across two packets), a REAL TLS ClientHello (made by Python's
ssl module through memory BIOs, so the SNI parser reads what a browser sends), and TLS application-data records that look like the real thing
(a record header, then random bytes: the analyser only reads headers and searches bytes). Checksums are zero: nothing here verifies them."""
from __future__ import annotations
import os
import socket
import ssl
import struct
ETHERNET, RAW, SLL2 = 1, 101, 276
SYN, ACK, PSH, FIN = 0x02, 0x10, 0x08, 0x01
def _ip4(src: str, dst: str, payload: bytes) -> bytes:
header = struct.pack(">BBHHHBBH4s4s", 0x45, 0, 20 + len(payload), 0, 0x4000, 64, 6, 0, socket.inet_aton(src), socket.inet_aton(dst))
return header + payload
def _ip6(src: str, dst: str, payload: bytes) -> bytes:
header = struct.pack(">IHBB16s16s", 0x60000000, len(payload), 6, 64, socket.inet_pton(socket.AF_INET6, src), socket.inet_pton(socket.AF_INET6, dst))
return header + payload
def tcp_frame(src: str, sport: int, dst: str, dport: int, seq: int, flags: int, payload: bytes = b"", *, link: int = ETHERNET) -> bytes:
tcp = struct.pack(">HHIIBBHHH", sport, dport, seq & 0xFFFFFFFF, 0, 5 << 4, flags, 65535, 0, 0) + payload
v6 = ":" in src
ip = _ip6(src, dst, tcp) if v6 else _ip4(src, dst, tcp)
kind = 0x86DD if v6 else 0x0800
if link == ETHERNET:
return b"\x02\x00\x00\x00\x00\x01" + b"\x02\x00\x00\x00\x00\x02" + struct.pack(">H", kind) + ip
if link == SLL2:
return struct.pack(">HHIHBB8s", kind, 0, 1, 1, 0, 6, b"\x02\x00\x00\x00\x00\x01\x00\x00") + ip
return ip
def pcap_file(frames: list, link: int = ETHERNET, endian: str = "<", nano: bool = False) -> bytes:
magic = {("<", False): b"\xd4\xc3\xb2\xa1", (">", False): b"\xa1\xb2\xc3\xd4", ("<", True): b"\x4d\x3c\xb2\xa1", (">", True): b"\xa1\xb2\x3c\x4d"}[(endian, nano)]
out = bytearray(magic + struct.pack(endian + "HHIIII", 2, 4, 0, 0, 262144, link))
for index, frame in enumerate(frames):
out += struct.pack(endian + "IIII", 1700000000 + index, 0, len(frame), len(frame)) + frame
return bytes(out)
class Conversation:
"""One TCP connection, client -> server, with a handshake and segments cut wherever the test says."""
def __init__(self, client=("203.0.113.9", 51000), server=("10.120.1.4", 443), *, link: int = ETHERNET, c_seq: int = 1000, s_seq: int = 9000):
self.client, self.server, self.link = client, server, link
self.c_seq, self.s_seq = c_seq, s_seq
self.frames: list = []
def _frame(self, from_client: bool, flags: int, payload: bytes = b"") -> None:
src, dst = (self.client, self.server) if from_client else (self.server, self.client)
seq = self.c_seq if from_client else self.s_seq
self.frames.append(tcp_frame(src[0], src[1], dst[0], dst[1], seq, flags, payload, link=self.link))
consumed = len(payload) + (1 if flags & (SYN | FIN) else 0)
if from_client:
self.c_seq += consumed
else:
self.s_seq += consumed
def handshake(self) -> "Conversation":
self._frame(True, SYN)
self._frame(False, SYN | ACK)
self._frame(True, ACK)
return self
def send(self, data: bytes, *, from_client: bool = True, cuts: tuple = ()) -> "Conversation":
"""Sends data in segments cut at the given offsets (default: one segment per 1400 bytes)."""
points = sorted(set(cuts)) or list(range(1400, len(data), 1400))
start = 0
for point in [*points, len(data)]:
if point > start:
self._frame(from_client, PSH | ACK, data[start:point])
start = point
return self
def reorder(self, first: int, second: int) -> "Conversation":
self.frames[first], self.frames[second] = self.frames[second], self.frames[first]
return self
def duplicate(self, index: int) -> "Conversation":
self.frames.append(self.frames[index])
return self
def pcap(self, **kwargs) -> bytes:
return pcap_file(self.frames, self.link, **kwargs)
def client_hello(server_name: str) -> bytes:
"""The first flight of a real TLS client (Python's ssl through memory BIOs) for the given SNI name."""
context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
incoming, outgoing = ssl.MemoryBIO(), ssl.MemoryBIO()
tls = context.wrap_bio(incoming, outgoing, server_hostname=server_name)
try:
tls.do_handshake()
except ssl.SSLWantReadError:
pass
return outgoing.read()
def app_data_records(total: int, *, avoid: tuple = ()) -> bytes:
"""TLS 1.2-style application-data records (type 23) of random bytes, `total` bytes of ciphertext in all."""
out = bytearray()
remaining = total
while remaining > 0:
size = min(remaining, 1200)
while True:
body = os.urandom(size)
if not any(marker in body for marker in avoid):
break
out += b"\x17\x03\x03" + struct.pack(">H", size) + body
remaining -= size
return bytes(out)
files/tunnel/.gitignore 4 lines · 118 B
# build outputs (build.sh writes to dist/; plain `go build` writes ./tunnel-server)
/tunnel-server
/all-tunnel
/dist/
files/tunnel/acme.go 137 lines · 3.8 KB
package main
import (
"bufio"
"bytes"
"io"
"net"
"net/http"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"time"
)
// Port 80 serves ONE thing: the HTTP-01 challenge files of the node's own certificate, from the webroot its certificate client writes. It never
// forwards a byte to a device (a certificate for a device is issued over DNS-01 and the CAA allows nothing else), it answers only for the node's own
// name, and everything else is a 404.
const (
maxACMEHeadBytes = 8 << 10
resp404 = "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
)
var acmePathRE = regexp.MustCompile(`^/\.well-known/acme-challenge/([A-Za-z0-9_-]{1,128})$`)
// hostOnly strips an optional :port and normalizes a Host header value.
func hostOnly(h string) (string, bool) {
if i := strings.LastIndexByte(h, ':'); i >= 0 && !strings.Contains(h[i:], "]") {
h = h[:i]
}
return normalizeName(h)
}
// readHTTPHead reads up to the end of the request head (CRLF CRLF).
func readHTTPHead(c net.Conn, timeout time.Duration) ([]byte, error) {
_ = c.SetReadDeadline(time.Now().Add(timeout))
defer func() { _ = c.SetReadDeadline(time.Time{}) }()
buf := make([]byte, 0, 1024)
searched := 0
for {
if len(buf) == cap(buf) {
if cap(buf) >= maxACMEHeadBytes {
return nil, errHelloTooLarge
}
grown := make([]byte, len(buf), min(2*cap(buf), maxACMEHeadBytes))
copy(grown, buf)
buf = grown
}
n, err := c.Read(buf[len(buf):cap(buf)])
buf = buf[:len(buf)+n]
from := max(0, searched-3)
if i := bytes.Index(buf[from:], []byte("\r\n\r\n")); i >= 0 {
return buf[:from+i+4], nil
}
searched = len(buf)
if err != nil {
return nil, err
}
}
}
// acmeChallenge reads the challenge file of a token from the webroot: a regular file (never a link) no larger than the cap.
func (s *Server) acmeChallenge(token string) ([]byte, bool) {
if s.cfg.ACMEWebroot == "" {
return nil, false
}
path := filepath.Join(s.cfg.ACMEWebroot, ".well-known", "acme-challenge", token)
st, err := os.Lstat(path)
if err != nil || !st.Mode().IsRegular() || st.Size() > s.cfg.ACMEMaxBytes {
return nil, false
}
f, err := os.Open(path)
if err != nil {
return nil, false
}
defer f.Close()
body, err := io.ReadAll(io.LimitReader(f, s.cfg.ACMEMaxBytes+1))
if err != nil || int64(len(body)) > s.cfg.ACMEMaxBytes {
return nil, false
}
return body, true
}
// handleHTTP serves only GET/HEAD /.well-known/acme-challenge/<token> for the node's own name; everything else is a 404.
func (s *Server) handleHTTP(c net.Conn) {
defer s.wg.Done()
defer c.Close()
// The same sources, the same limits as :443: a source that floods one port does not get a second budget on the other.
if peer, ok, fromOpenAI := s.sourceOf(c); ok {
release, why := s.ips.admit(peer, fromOpenAI)
if why != "" {
s.met.rejected.inc(why)
return
}
defer release()
}
if !tryAcquire(&s.unauth, int64(s.cfg.MaxUnauth)) {
return
}
defer s.unauth.Add(-1)
head, err := readHTTPHead(c, s.cfg.HelloTimeout)
if err != nil {
return
}
req, err := http.ReadRequest(bufio.NewReader(bytes.NewReader(head)))
if err != nil {
return
}
_ = c.SetWriteDeadline(time.Now().Add(s.cfg.ACMETimeout))
reply404 := func(reason string) {
s.log.Debug("acme_reject", "reason", reason)
_, _ = io.WriteString(c, resp404)
}
host, hostOK := hostOnly(req.Host)
if !hostOK || host != s.cfg.NodeHost {
reply404("not_own_host")
return
}
m := acmePathRE.FindStringSubmatch(req.RequestURI)
if (req.Method != http.MethodGet && req.Method != http.MethodHead) || m == nil {
reply404("not_acme_path")
return
}
body, found := s.acmeChallenge(m[1])
if !found {
reply404("no_challenge")
return
}
_, _ = io.WriteString(c, "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: "+strconv.Itoa(len(body))+"\r\nConnection: close\r\n\r\n")
if req.Method == http.MethodGet {
_, _ = c.Write(body)
}
}
files/tunnel/auth.go 479 lines · 16.8 KB
package main
import (
"context"
"crypto/ed25519"
"crypto/sha256"
"fmt"
"strconv"
"strings"
"sync"
"time"
)
// Device authentication of protocol v2 (the contract, section 3): the node verifies the device's session JWT offline against the control
// plane's keys, makes the device prove it holds the session's key (a DPoP proof over the NODE's URL and a nonce the node just made), and then asks
// the control plane what the device is allowed: the host, the status and how long. Nothing a device sends names a host.
// A token problem is one closed word. The words and their order are those of the contracts' verifiers (tests run every golden vector through
// the functions below).
const (
tokMalformed = "malformed_token"
tokBadAlg = "bad_alg"
tokUnknownKID = "unknown_kid"
tokBadSignature = "bad_signature"
tokWrongType = "wrong_type"
tokWrongIssuer = "wrong_issuer"
tokWrongAudience = "wrong_audience"
tokWrongKey = "wrong_key"
tokWrongSubject = "wrong_subject"
tokWrongHTM = "wrong_htm"
tokWrongHTU = "wrong_htu"
tokWrongATH = "wrong_ath"
tokWrongNonce = "wrong_nonce"
tokBadClaims = "bad_claims"
tokBadLifetime = "bad_lifetime"
tokNotYetValid = "not_yet_valid"
tokExpired = "expired"
tokStaleProof = "stale_proof"
tokReplayedJTI = "replayed_jti"
)
type tokenError struct{ word string }
func (e *tokenError) Error() string { return e.word }
func tokFail(word string) error { return &tokenError{word} }
// tokenWord is the closed word of a token error ("" when err is nil).
func tokenWord(err error) string {
if err == nil {
return ""
}
if te, ok := err.(*tokenError); ok {
return te.word
}
return tokMalformed
}
// errFor turns a token word into the ERR a refused HELLO, RENEW or PROOF is answered with, as the reference node of the contracts does: a clock that
// is off is clock_skew (the device shows it), a malformed token is bad_hello, a token for another audience or an expired one is auth, and every other
// word is the fallback (auth for a session, bad_proof for a proof) because nothing else may be told to a device that failed.
func errFor(word, fallback string) string {
switch word {
case tokMalformed:
return codeBadHello
case tokNotYetValid, tokStaleProof:
return codeClockSkew
case tokWrongAudience, tokExpired:
return codeAuth
}
return fallback
}
type jws struct {
header, claims map[string]any
signature []byte
signingInput []byte
}
// splitJWS splits and decodes a compact JWS. Any structural fault is malformed_token: ASCII only, three dot-separated base64url segments with no
// padding and canonical encoding, header and claims JSON objects without a repeated key and without a fraction, none of the headers that change
// how a signature is read (crit, b64, jku, x5u, x5c). An empty signature part is allowed here (alg none then fails as bad_alg, the more useful word).
func splitJWS(token string, maxBytes int) (*jws, error) {
if len(token) > maxBytes {
return nil, tokFail(tokMalformed)
}
for i := 0; i < len(token); i++ {
if token[i] > 0x7f {
return nil, tokFail(tokMalformed)
}
}
parts := strings.Split(token, ".")
if len(parts) != 3 {
return nil, tokFail(tokMalformed)
}
decode := func(text string) (map[string]any, bool) {
raw, err := b64Decode(text)
if err != nil {
return nil, false
}
value, err := parseJSON(string(raw))
if err != nil {
return nil, false
}
object, ok := value.(map[string]any)
return object, ok
}
header, ok1 := decode(parts[0])
claims, ok2 := decode(parts[1])
if !ok1 || !ok2 {
return nil, tokFail(tokMalformed)
}
var signature []byte
if parts[2] != "" {
raw, err := b64Decode(parts[2])
if err != nil {
return nil, tokFail(tokMalformed)
}
signature = raw
}
for _, name := range []string{"crit", "b64", "jku", "x5u", "x5c"} {
if _, present := header[name]; present {
return nil, tokFail(tokMalformed)
}
}
return &jws{header, claims, signature, []byte(parts[0] + "." + parts[1])}, nil
}
func verifyEd25519(key ed25519.PublicKey, signature, input []byte) bool {
return len(signature) == ed25519.SignatureSize && len(key) == ed25519.PublicKeySize && ed25519.Verify(key, input, signature)
}
func claimInt(claims map[string]any, key string) (int64, bool) {
f, ok := claims[key].(float64)
if !ok || f != float64(int64(f)) {
return 0, false
}
return int64(f), true
}
func claimString(claims map[string]any, key string) (string, bool) {
s, ok := claims[key].(string)
return s, ok
}
// epochClaim reads an epoch-seconds claim in the contracts' range.
func epochClaim(claims map[string]any, key string) (int64, bool) {
n, ok := epochOK(claims[key])
return n, ok
}
// SessionClaims are the claims of a verified device session the node keeps for the life of a connection.
type SessionClaims struct {
Subject string // the device id
JTI string
Generation int64
IssuedAt int64
ExpiresAt int64
KeyX string // the session's bound key (cnf.jwk.x): the key a proof must be signed with
}
func epochSeconds(t time.Time) float64 { return float64(t.UnixNano()) / 1e9 }
// publicJWKOK is the contracts' public JWK: OKP, Ed25519, an x of 43 canonical characters, no private part.
func publicJWKX(value any) (string, bool) {
jwk, ok := value.(map[string]any)
if !ok {
return "", false
}
x, hasX := jwk["x"].(string)
if _, private := jwk["d"]; private || jwk["kty"] != "OKP" || jwk["crv"] != "Ed25519" || !hasX {
return "", false
}
if _, ok := publicKeyX(x); !ok {
return "", false
}
return x, true
}
// verifySession verifies a device session offline: the signature before any claim is trusted, then the audience (the node accepts only
// all-tunnel: the relay's and the control plane's audiences and a missing audience are refused), the lifetime (exactly the policy's session TTL) and the clock within skew.
// The words, in the order the checks fail: malformed_token, bad_alg, unknown_kid, bad_signature, wrong_type, wrong_issuer, wrong_audience,
// bad_claims, bad_lifetime, not_yet_valid, expired.
func verifySession(token string, now time.Time, keys KeySource, skew time.Duration) (*SessionClaims, error) {
if len(token) > policyData.Protocol.JWTMaxBytes {
return nil, tokFail(tokMalformed)
}
j, err := splitJWS(token, policyData.Protocol.JWTMaxBytes)
if err != nil {
return nil, err
}
if j.header["alg"] != "EdDSA" {
return nil, tokFail(tokBadAlg)
}
kid, isString := j.header["kid"].(string)
if !isString || !reKID.MatchString(kid) {
return nil, tokFail(tokUnknownKID)
}
key, known := keys.Key(kid)
if !known {
return nil, tokFail(tokUnknownKID)
}
if !verifyEd25519(key, j.signature, j.signingInput) {
return nil, tokFail(tokBadSignature)
}
c := j.claims
if j.header["typ"] != "device-session" || c["typ"] != "device-session" {
return nil, tokFail(tokWrongType)
}
if ver, ok := claimInt(c, "ver"); !ok || ver != int64(policyData.Protocol.Version) {
return nil, tokFail(tokWrongType)
}
if c["iss"] != policyData.Identity.Issuer {
return nil, tokFail(tokWrongIssuer)
}
if c["aud"] != policyData.Identity.AudTunnel {
return nil, tokFail(tokWrongAudience)
}
sub, subOK := claimString(c, "sub")
jti, jtiOK := claimString(c, "jti")
iat, iatOK := epochClaim(c, "iat")
exp, expOK := epochClaim(c, "exp")
generation, genOK := claimInt(c, "generation")
x, cnfOK := "", false
if cnf, ok := c["cnf"].(map[string]any); ok && len(cnf) == 1 {
if jwk, ok := cnf["jwk"].(map[string]any); ok && len(jwk) == 3 {
x, cnfOK = publicJWKX(jwk)
}
}
if !subOK || !reDeviceID.MatchString(sub) || !jtiOK || !reJTI.MatchString(jti) || !iatOK || !expOK || !genOK || generation < 1 || generation > 1<<53-1 || !cnfOK {
return nil, tokFail(tokBadClaims)
}
if exp-iat != int64(policyData.Authorization.SessionTTLS) {
return nil, tokFail(tokBadLifetime)
}
t := epochSeconds(now)
if float64(iat) > t+skew.Seconds() {
return nil, tokFail(tokNotYetValid)
}
if float64(exp) <= t {
return nil, tokFail(tokExpired)
}
return &SessionClaims{Subject: sub, JTI: jti, Generation: generation, IssuedAt: iat, ExpiresAt: exp, KeyX: x}, nil
}
func sha256B64(data []byte) string {
sum := sha256.Sum256(data)
return b64Encode(sum[:])
}
// nodeProofURL is the htu a proof must carry: https://<node host>[:<port> when it is not 443]/all-tunnel/2.
func nodeProofURL(host string, port int) string {
u := "https://" + host
if port != 443 {
u += ":" + strconv.Itoa(port)
}
return u + policyData.Protocol.ProofHTUPath
}
// proofClaims is what a verified proof leaves the node: the jti, to be spent once.
type proofClaims struct {
JTI string
IssuedAt int64
}
// verifyProof verifies the DPoP proof of the session the connection said it holds: signed by the session's bound key, for this session (ath),
// this node's URL (htu), this exchange (the nonce the node made) and this moment (iat within the policy's proof TTL). The jti is NOT spent here: the
// caller spends it (replayCache.spend) so that a replay is refused even between two connections. Words, in order: malformed_token, bad_alg,
// wrong_type, wrong_key, bad_signature, wrong_subject, wrong_audience, wrong_htm, wrong_htu, wrong_ath, wrong_nonce, bad_claims, not_yet_valid,
// stale_proof.
func verifyProof(proof string, now time.Time, sessionToken string, session *SessionClaims, nonce, htu string, skew time.Duration) (*proofClaims, error) {
if len(proof) > policyData.Protocol.JWTMaxBytes {
return nil, tokFail(tokMalformed)
}
j, err := splitJWS(proof, policyData.Protocol.JWTMaxBytes)
if err != nil {
return nil, err
}
if j.header["alg"] != "EdDSA" {
return nil, tokFail(tokBadAlg)
}
if j.header["typ"] != "dpop+jwt" {
return nil, tokFail(tokWrongType)
}
x, ok := publicJWKX(j.header["jwk"])
if !ok {
return nil, tokFail(tokMalformed)
}
if x != session.KeyX {
return nil, tokFail(tokWrongKey)
}
key, _ := publicKeyX(x)
if !verifyEd25519(key, j.signature, j.signingInput) {
return nil, tokFail(tokBadSignature)
}
c := j.claims
switch {
case c["sub"] != session.Subject:
return nil, tokFail(tokWrongSubject)
case c["aud"] != policyData.Identity.AudTunnel:
return nil, tokFail(tokWrongAudience)
case c["htm"] != policyData.Protocol.ProofHTM:
return nil, tokFail(tokWrongHTM)
case c["htu"] != htu:
return nil, tokFail(tokWrongHTU)
case c["ath"] != sha256B64([]byte(sessionToken)):
return nil, tokFail(tokWrongATH)
case c["nonce"] != nonce:
return nil, tokFail(tokWrongNonce)
}
jti, jtiOK := claimString(c, "jti")
iat, iatOK := epochClaim(c, "iat")
if !jtiOK || !reJTI.MatchString(jti) || !iatOK {
return nil, tokFail(tokBadClaims)
}
t := epochSeconds(now)
if float64(iat) > t+skew.Seconds() {
return nil, tokFail(tokNotYetValid)
}
if float64(iat+int64(policyData.Authorization.ProofTTLS)) <= t {
return nil, tokFail(tokStaleProof)
}
return &proofClaims{JTI: jti, IssuedAt: iat}, nil
}
// replayCache remembers the proofs the node has accepted until they could not verify any more (a proof is single use for the policy's proof TTL,
// plus the skew a fast clock is allowed). A session's jti is NOT single use: a cached session reconnects. The cache is bounded: when it is full of
// live entries a new proof is refused (the connection gets bad_proof), never admitted unchecked.
type replayCache struct {
mu sync.Mutex
seen map[string]time.Time // jti -> when it can be forgotten
limit int
}
func newReplayCache(limit int) *replayCache {
return &replayCache{seen: map[string]time.Time{}, limit: limit}
}
// spend records a proof jti issued at iat. It reports false when the jti was already spent (or the cache is full of live entries).
func (r *replayCache) spend(jti string, iat int64, now time.Time, skew time.Duration) bool {
forget := time.Unix(iat, 0).Add(secs(policyData.Authorization.ProofTTLS) + skew)
r.mu.Lock()
defer r.mu.Unlock()
if until, dup := r.seen[jti]; dup && now.Before(until) {
return false
}
if len(r.seen) >= r.limit {
for id, until := range r.seen {
if !now.Before(until) {
delete(r.seen, id)
}
}
if len(r.seen) >= r.limit {
return false
}
}
r.seen[jti] = forget
return true
}
func (r *replayCache) size() int {
r.mu.Lock()
defer r.mu.Unlock()
return len(r.seen)
}
// ControlPlane is the node's side of POST /v1/tunnel/authorize: what the control plane decides for a device whose session the node verified. The
// request never carries a host name; the answer's host, status and valid_until are what the node relays. The real client (node assertion, retry,
// concurrency cap, grace) is the control-plane piece; the node core is written against this interface and tested with a fake.
type ControlPlane interface {
Authorize(ctx context.Context, req AuthorizeRequest) (AuthorizeResponse, error)
}
// AuthorizeRequest is authorize-request.schema.json.
type AuthorizeRequest struct {
DeviceID string
SessionJTI string
Reason string // "hello" or "renew"
Node string
}
func (r AuthorizeRequest) toMap() map[string]any {
return map[string]any{"v": 1, "device_id": r.DeviceID, "session_jti": r.SessionJTI, "reason": r.Reason, "node": r.Node}
}
// AuthorizeResponse is the 200 answer of authorize (authorize-response.schema.json): the node uses Host, Status, ValidUntil, ServerTime and the
// Policy windows; the rest travels to the pieces that need it.
type AuthorizeResponse struct {
DeviceID string
Label string
Host string
Status string // active, gated or paused
Plan string // paid or trial
ValidUntil int64 // epoch seconds
ServerTime int64
Policy okPolicy
Access string // the signed access statement, when the answer has one
Nodes any // the signed list of nodes, opaque to this piece
E2EWave int
E2EKill bool
CopyE2EE bool
}
func (a AuthorizeResponse) toMap() map[string]any {
m := map[string]any{"ok": true, "v": 1, "device_id": a.DeviceID, "label": a.Label, "host": a.Host, "status": a.Status, "plan": a.Plan,
"valid_until": a.ValidUntil, "server_time": a.ServerTime, "policy": a.Policy.toMap(), "e2e_wave": a.E2EWave, "e2e_kill": a.E2EKill,
"copy_e2ee": a.CopyE2EE}
if a.Nodes != nil {
m["nodes"] = a.Nodes
}
if a.Access != "" {
m["access"] = a.Access
}
return m
}
// ControlError is a refusal of authorize: the HTTP status and the closed word of the error body (license_inactive, device_revoked, ...). A transport
// failure or a timeout is any other error: the node answers control_unavailable.
type ControlError struct {
Status int
Code string
}
func (e *ControlError) Error() string { return fmt.Sprintf("authorize: %d %s", e.Status, e.Code) }
// errForControl is the ERR word of a failed authorize.
func errForControl(err error) string {
ce, ok := err.(*ControlError)
if !ok {
return codeControlUnavailable
}
if ce.Status >= 500 || ce.Status == 429 || ce.Status < 400 {
return codeControlUnavailable
}
switch ce.Code {
case codeLicenseInactive, codeDeviceRevoked, codePaused:
return ce.Code
case "unauthorized", "invalid_authorize_request", "request_too_large", "bad_response":
// The control plane refused the NODE (its assertion, its request), not the device: nothing the device can fix, so "try later", never "auth".
return codeControlUnavailable
}
return codeAuth
}
// checkAuthorize refuses an answer that is not what was asked: another device's, a host that is not exactly one label under a zone or does not start
// with the answer's own label, an unknown status or plan, windows out of range. The node never relays an answer it cannot vouch for.
func checkAuthorize(a AuthorizeResponse, req AuthorizeRequest) error {
switch {
case a.DeviceID != req.DeviceID:
return fmt.Errorf("authorize answer is for another device")
case !reLabel.MatchString(a.Label), !validFQDN(a.Host), !strings.HasPrefix(a.Host, a.Label+"."):
return fmt.Errorf("authorize answer has a bad host")
case !validStatus(a.Status):
return fmt.Errorf("authorize answer has a bad status")
case a.Plan != "paid" && a.Plan != "trial":
return fmt.Errorf("authorize answer has a bad plan")
case a.ValidUntil < 1 || a.ValidUntil > epochMax || a.ServerTime < 1 || a.ServerTime > epochMax:
return fmt.Errorf("authorize answer has a bad time")
}
for _, v := range []int64{a.Policy.PingIntervalS, a.Policy.DeadPeerS, a.Policy.StreamIdleS, a.Policy.StreamMaxAgeS, a.Policy.RenewBeforeValidUntilS} {
if v < 1 || v > 86400 {
return fmt.Errorf("authorize answer has a bad policy window")
}
}
return nil
}
// boundWindows bounds the windows of the answer to policy.json: the node never offers a device a window longer than the table's own. (The control
// plane builds them from the same table, so this changes nothing for an honest answer.)
func boundWindows(p okPolicy) okPolicy {
t := policyData.Timing
return okPolicy{
PingIntervalS: min(p.PingIntervalS, int64(t.PingIntervalS)),
DeadPeerS: min(p.DeadPeerS, int64(t.DeadPeerS)),
StreamIdleS: min(p.StreamIdleS, int64(t.StreamIdleS)),
StreamMaxAgeS: min(p.StreamMaxAgeS, int64(t.StreamMaxAgeS)),
RenewBeforeValidUntilS: min(p.RenewBeforeValidUntilS, int64(t.RenewBeforeValidUntilS)),
}
}
files/tunnel/backend_prod.go 71 lines · 2.9 KB
//go:build !testnode && !testcp
package main
import (
"flag"
"fmt"
"io"
)
// The production backend: the node's real control-plane client, JWKS cache and feed. The node holds exactly one secret, its own Ed25519 signing key
// (-node-key-file, mode 0600); the control plane holds the public half in TUNNEL_NODE_KEYS and a node that is taken over is revoked by removing its
// kid there. There is no shared secret to configure, and the control plane's address is the issuer the policy pins unless an operator points a
// staging node somewhere else on purpose.
func init() {
theBackend = &backend{flags: prodFlags}
defaultMetricsAddr = "127.0.0.1:9101"
defaultOpenAIPrefixesURL = "https://openai.com/chatgpt-connectors.json"
}
func prodFlags(fs *flag.FlagSet, stdout io.Writer) func(*Config) error {
controlURL := fs.String("control-url", envStr("TUNNEL_CONTROL_URL", policyData.Identity.Issuer), "the control plane (https://host[:port]): authorize, the feed and the published keys come from here only [TUNNEL_CONTROL_URL]")
keyFile := fs.String("node-key-file", envStr("TUNNEL_NODE_KEY_FILE", ""), "this node's Ed25519 signing key (PKCS#8 PEM, mode 0600): the one credential this node has [TUNNEL_NODE_KEY_FILE]")
kid := fs.String("node-kid", envStr("TUNNEL_NODE_KID", ""), "the kid of this node's public key in the control plane's TUNNEL_NODE_KEYS [TUNNEL_NODE_KID]")
initKey := fs.Bool("node-key-init", false, "create the signing key at -node-key-file (never over an existing one), print the TUNNEL_NODE_KEYS entry that registers it, and exit")
printKey := fs.Bool("node-key-print", false, "print the TUNNEL_NODE_KEYS entry of the key at -node-key-file and exit")
return func(cfg *Config) error {
if *keyFile == "" {
return fmt.Errorf("the node needs its signing key: -node-key-file (create it once with -node-key-init)")
}
if (*initKey || *printKey) && (!reNodeID.MatchString(cfg.NodeID) || !reKID.MatchString(*kid)) {
return fmt.Errorf("the TUNNEL_NODE_KEYS entry needs -node-id and -node-kid")
}
if *initKey {
key, err := GenerateNodeKey(*keyFile)
if err != nil {
return err
}
fmt.Fprintln(stdout, NodeKeysEntry(cfg.NodeID, *kid, key))
return errDone
}
key, err := LoadNodeKey(*keyFile)
if err != nil {
return err
}
if *printKey {
fmt.Fprintln(stdout, NodeKeysEntry(cfg.NodeID, *kid, key))
return errDone
}
control, err := NewHTTPControl(HTTPControlConfig{BaseURL: *controlURL, NodeID: cfg.NodeID, KID: *kid, Key: key, Version: version})
if err != nil {
return err
}
clk := newClock(cfg.Now, cfg.AfterFunc)
health := &cpHealth{}
jwks := NewJWKSCache(JWKSConfig{
Fetch: control.FetchJWKS, Clock: clk, Log: cfg.Logger,
Observe: func(ok bool) {
if ok {
health.good(clk.Now())
} else {
health.bad(clk.Now())
}
},
})
cfg.Keys, cfg.Control, cfg.Feed, cfg.Health = jwks, control, control, health
cfg.Background = append(cfg.Background, jwks.Run)
return nil
}
}
files/tunnel/build.sh 74 lines · 4.0 KB
#!/usr/bin/env bash
# Build the tunnel node into dist/ (standard library only, no cgo).
#
# build.sh [node|test-node|blind-mutant|blind-mutant-vm|chaos-node|loadgen|all] (default: all)
#
# node cross-compile the production node for the Ubuntu 24.04 VM (linux/amd64, static) and print its SHA-256
# chaos-node [OUT [POLICY]] build the CHAOS NODE (-tags testcp: the production control-plane client against a loopback control plane, see backend_loopcp.go)
# for THIS machine into OUT (default dist/all-tunnel-chaosnode). POLICY, a policy.json with the long windows shortened, replaces the embedded one
# for this build only (go build -overlay), which is how the chaos suite compresses hours into seconds. Never deployed.
# loadgen [OUT] build the load and chaos generator (cmd/loadgen) for THIS machine into OUT (default dist/loadgen)
# test-node build the TEST NODE for THIS machine (-tags testnode: file-driven fakes for the control plane's keys and answers, see testnode.go).
# It is what ALL_TEST_NODE_BIN names for the device's tests:
# export ALL_TEST_NODE_BIN="$PWD/dist/all-tunnel-testnode"
# It is never deployed.
# blind-mutant-vm the same mutant cross-compiled for the gate's VMs (no go toolchain there): dist/all-tunnel-blindmutant-linux-amd64 and dist/all-tunnel-blindmutant-windows.exe.
# The gate picks them up from dist (or ALL_TEST_BLIND_MUTANT_BIN_LINUX / _WINDOWS) so the blindness selftest runs on the VMs too.
# blind-mutant the test node plus ONE test-only change: it terminates a client's TLS itself and logs the request in the clear (blindmutant_test_hook.go).
# It is the node that the blindness gate must catch; it is not part of `all` and is never deployed.
set -euo pipefail
cd "$(dirname "$0")"
mkdir -p dist
version="$(git rev-parse --short HEAD 2>/dev/null || echo dev)"
what="${1:-all}"
node() {
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \
go build -trimpath -ldflags "-s -w -X main.version=${version}" -o dist/all-tunnel-linux-amd64 .
shasum -a 256 dist/all-tunnel-linux-amd64 | tee dist/all-tunnel-linux-amd64.sha256
}
test_node() {
CGO_ENABLED=0 go build -trimpath -tags testnode -ldflags "-X main.version=${version}-testnode" -o dist/all-tunnel-testnode .
echo "$PWD/dist/all-tunnel-testnode"
}
blind_mutant() {
CGO_ENABLED=0 go build -trimpath -tags "testnode blindmutant" -ldflags "-X main.version=${version}-blindmutant" -o dist/all-tunnel-blindmutant .
echo "$PWD/dist/all-tunnel-blindmutant"
}
blind_mutant_vm() {
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -tags "testnode blindmutant" -ldflags "-X main.version=${version}-blindmutant" -o dist/all-tunnel-blindmutant-linux-amd64 .
CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -trimpath -tags "testnode blindmutant" -ldflags "-X main.version=${version}-blindmutant" -o dist/all-tunnel-blindmutant-windows.exe .
shasum -a 256 dist/all-tunnel-blindmutant-linux-amd64 dist/all-tunnel-blindmutant-windows.exe
}
chaos_node() {
local out="${1:-dist/all-tunnel-chaosnode}" policy="${2:-}" overlay=() file=""
trap '[ -z "$file" ] || rm -f "$file"' RETURN
if [ -n "$policy" ]; then
file="$(mktemp)"
printf '{"Replace":{"%s/policy.json":"%s"}}' "$PWD" "$(cd "$(dirname "$policy")" && pwd)/$(basename "$policy")" > "$file"
overlay=(-overlay "$file")
fi
CGO_ENABLED=0 go build -trimpath -tags testcp ${overlay[@]+"${overlay[@]}"} -ldflags "-X main.version=${version}-chaos" -o "$out" .
echo "$out"
}
loadgen() {
local out="${1:-dist/loadgen}"
CGO_ENABLED=0 go build -trimpath -o "$out" ./cmd/loadgen
echo "$out"
}
case "$what" in
node) node ;;
test-node) test_node ;;
blind-mutant) blind_mutant ;;
blind-mutant-vm) blind_mutant_vm ;;
chaos-node) shift; chaos_node "$@" ;;
loadgen) shift; loadgen "$@" ;;
all) node; test_node ;;
*) echo "usage: build.sh [node|test-node|blind-mutant|blind-mutant-vm|chaos-node [OUT [POLICY]]|loadgen [OUT]|all]" >&2; exit 2 ;;
esac
files/tunnel/certreload.go 192 lines · 6.3 KB
package main
import (
"context"
"crypto/tls"
"crypto/x509"
"errors"
"fmt"
"log/slog"
"os"
"sync"
"time"
)
// The node's own public-CA certificate (the one devices verify on the node's own name, ALPN all-tunnel/2) is renewed by the node's own ACME client
// (certbot, HTTP-01 through the webroot on :80) and replaced on disk. certReloader puts the new one in service without a restart and without a dropped
// handshake: it reads the files at start and again whenever they change, at a handshake or on its own timer, and a renewal that is not good (half
// written, a key that does not match, another name, already expired) never replaces the certificate that works.
// fileSig is what the node looks at to know a file changed: its modification time and size (certbot's live files are symlinks into an archive, and Stat
// follows them, so a renewal that moves the link is a different file).
type fileSig struct{ mod, size int64 }
func sigOf(path string) (fileSig, error) {
st, err := os.Stat(path)
if err != nil {
return fileSig{}, err
}
return fileSig{st.ModTime().UnixNano(), st.Size()}, nil
}
const (
certLookGap = time.Second // a handshake looks at the files at most this often (certReloader.lookGap)
certRetryGap = 30 * time.Second // a renewal that did not load is tried again after this even if its files did not change again
certWatchEvery = time.Minute // the node looks at the files by itself this often
certWarnDays = 5 // days before the end at which the log says it loudly
)
type certReloader struct {
certPath, keyPath string
host string // the node's own name the certificate must be valid for; "" skips the name check (tests)
clk clock
log *slog.Logger
lookGap time.Duration
mu sync.Mutex
pair *tls.Certificate
notAfter time.Time
certSig fileSig // what the certificate in service was read from
keySig fileSig
lookedAt time.Time
failedAt time.Time
failedCrt fileSig // what the last refused renewal was read from
failedKey fileSig
lastErr string
loads int
}
func newCertReloader(certPath, keyPath, host string, clk clock, log *slog.Logger) (*certReloader, error) {
if clk.now == nil {
clk = newClock(nil, nil)
}
if log == nil {
log = slog.New(slog.DiscardHandler)
}
r := &certReloader{certPath: certPath, keyPath: keyPath, host: host, clk: clk, log: log, lookGap: certLookGap}
if err := r.load(); err != nil {
return nil, fmt.Errorf("node certificate: %w", err)
}
return r, nil
}
// load reads the pair, checks it and puts it in service. The caller holds no lock.
func (r *certReloader) load() error {
cs, cerr := sigOf(r.certPath)
ks, kerr := sigOf(r.keyPath)
if err := errors.Join(cerr, kerr); err != nil {
return err
}
pair, err := tls.LoadX509KeyPair(r.certPath, r.keyPath)
if err != nil {
return err
}
leaf, err := x509.ParseCertificate(pair.Certificate[0])
if err != nil {
return err
}
now := r.clk.Now()
if r.host != "" {
if err := leaf.VerifyHostname(r.host); err != nil {
return fmt.Errorf("the certificate is not for %s", r.host)
}
}
if !now.Before(leaf.NotAfter) {
return fmt.Errorf("the certificate expired at %s", leaf.NotAfter.UTC().Format(time.RFC3339))
}
if now.Add(time.Hour).Before(leaf.NotBefore) {
return fmt.Errorf("the certificate is not valid before %s", leaf.NotBefore.UTC().Format(time.RFC3339))
}
pair.Leaf = leaf
r.mu.Lock()
r.pair, r.notAfter, r.certSig, r.keySig = &pair, leaf.NotAfter, cs, ks
r.lastErr, r.failedAt, r.failedCrt, r.failedKey = "", time.Time{}, fileSig{}, fileSig{}
r.loads++
r.mu.Unlock()
return nil
}
// refresh looks at the files (at most every certLookGap unless force) and loads them when they are not the ones in service. A refused renewal is
// remembered by its files, so it is parsed again only when they change or certRetryGap passed.
func (r *certReloader) refresh(force bool) {
now := r.clk.Now()
r.mu.Lock()
if !force && now.Sub(r.lookedAt) < r.lookGap {
r.mu.Unlock()
return
}
r.lookedAt = now
cur, curKey, failedCrt, failedKey, failedAt := r.certSig, r.keySig, r.failedCrt, r.failedKey, r.failedAt
r.mu.Unlock()
cs, cerr := sigOf(r.certPath)
ks, kerr := sigOf(r.keyPath)
if cerr != nil || kerr != nil {
return // a renewal in the middle of replacing its files: the certificate in service stays
}
if cs == cur && ks == curKey {
return
}
if !force && cs == failedCrt && ks == failedKey && !failedAt.IsZero() && now.Sub(failedAt) < certRetryGap {
return
}
if err := r.load(); err != nil {
r.mu.Lock()
r.failedCrt, r.failedKey, r.failedAt, r.lastErr = cs, ks, now, err.Error()
r.mu.Unlock()
r.log.Warn("node_cert_reload_refused", "reason", err.Error())
return
}
r.log.Info("node_cert_reloaded", "not_after", r.NotAfter().UTC().Format(time.RFC3339))
}
// Reload loads the files now, whatever they look like (SIGHUP): it returns why a renewal was refused.
func (r *certReloader) Reload() error {
if err := r.load(); err != nil {
r.mu.Lock()
r.lastErr = err.Error()
r.mu.Unlock()
return err
}
return nil
}
// get is the Config.CertSource: the certificate in service, after a look at the files.
func (r *certReloader) get(*tls.ClientHelloInfo) (*tls.Certificate, error) {
r.refresh(false)
r.mu.Lock()
defer r.mu.Unlock()
return r.pair, nil
}
// NotAfter is the end of the certificate in service (the monitor alerts under 14 days and under 5).
func (r *certReloader) NotAfter() time.Time {
r.mu.Lock()
defer r.mu.Unlock()
return r.notAfter
}
// LastError is why the last renewal was refused ("" when the certificate in service is the newest on disk).
func (r *certReloader) LastError() string {
r.mu.Lock()
defer r.mu.Unlock()
return r.lastErr
}
// Run looks at the files on its own timer, so a renewal is in service even when no device is connecting, and says in the log how long the certificate
// has left (the provider's expiry e-mails no longer exist, the node's own log and the site's monitor are what warn).
func (r *certReloader) Run(ctx context.Context) {
for {
r.refresh(true)
left := r.NotAfter().Sub(r.clk.Now())
switch {
case left <= 0:
r.log.Error("node_cert_expired")
case left < certWarnDays*24*time.Hour:
r.log.Warn("node_cert_expires_soon", "days_left", int(left/(24*time.Hour)))
}
if !r.clk.sleep(ctx, certWatchEvery) {
return
}
}
}
files/tunnel/certreload_test.go 297 lines · 10.8 KB
package main
import (
"bufio"
"context"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"log/slog"
"math/big"
"net"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
// The node's own public-CA certificate is hot reloaded: a renewal goes into service without a restart and without a dropped connection, and a renewal
// that is not good never replaces the one that works.
type certFiles struct {
dir, cert, key string
}
func newCertFiles(t *testing.T) *certFiles {
dir := t.TempDir()
return &certFiles{dir: dir, cert: filepath.Join(dir, "fullchain.pem"), key: filepath.Join(dir, "privkey.pem")}
}
// write puts a certificate for names, valid between notBefore and notAfter, on disk (the key too, unless keyFrom is set: then that pair's key is
// written, which is a renewal whose key does not match). It returns the serial.
func (c *certFiles) write(t *testing.T, serial int64, notBefore, notAfter time.Time, names ...string) {
t.Helper()
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
t.Fatal(err)
}
tmpl := &x509.Certificate{
SerialNumber: big.NewInt(serial), Subject: pkix.Name{CommonName: names[0]}, NotBefore: notBefore, NotAfter: notAfter,
KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, DNSNames: names,
}
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
if err != nil {
t.Fatal(err)
}
keyDER, _ := x509.MarshalECPrivateKey(key)
c.put(t, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}))
}
// put writes both files the way a renewal tool does: each replaced atomically, the key after the certificate (so there is a moment when they do not match).
func (c *certFiles) put(t *testing.T, certPEM, keyPEM []byte) {
t.Helper()
for path, data := range map[string][]byte{c.cert: certPEM, c.key: keyPEM} {
tmp := path + ".new"
if err := os.WriteFile(tmp, data, 0o600); err != nil {
t.Fatal(err)
}
if err := os.Rename(tmp, path); err != nil {
t.Fatal(err)
}
}
}
func (c *certFiles) fresh(t *testing.T, serial int64, names ...string) {
t.Helper()
c.write(t, serial, time.Now().Add(-time.Hour), time.Now().Add(60*24*time.Hour), names...)
}
// servedSerial connects to the node's own name and reports the serial of the certificate it presents.
func servedSerial(t *testing.T, addr string) int64 {
t.Helper()
conn, err := tls.DialWithDialer(&net.Dialer{Timeout: 3 * time.Second}, "tcp", addr, &tls.Config{
ServerName: testNodeHost, InsecureSkipVerify: true, NextProtos: []string{policyData.Protocol.ALPN}, MinVersion: tls.VersionTLS13,
})
if err != nil {
t.Fatal(err)
}
defer conn.Close()
return conn.ConnectionState().PeerCertificates[0].SerialNumber.Int64()
}
func certNode(t *testing.T, r *certReloader) *testEnv {
return newTestEnv(t, func(c *Config) { c.CertSource = r.get })
}
func TestCertReload_ARenewalIsInServiceForTheNextHandshakeAndAnOpenConnectionIsUntouched(t *testing.T) {
files := newCertFiles(t)
files.fresh(t, 1, testNodeHost)
r, err := newCertReloader(files.cert, files.key, testNodeHost, clock{}, nil)
if err != nil {
t.Fatal(err)
}
r.lookGap = 0
e := certNode(t, r)
if got := servedSerial(t, e.httpsAddr); got != 1 {
t.Fatalf("served %d", got)
}
// a device is connected (its TLS session was made with the first certificate)
d := e.newDevice()
conn, err := tls.DialWithDialer(&net.Dialer{Timeout: 3 * time.Second}, "tcp", e.httpsAddr, &tls.Config{
ServerName: testNodeHost, InsecureSkipVerify: true, NextProtos: []string{policyData.Protocol.ALPN}, MinVersion: tls.VersionTLS13,
})
if err != nil {
t.Fatal(err)
}
defer conn.Close()
dc := &devConn{t: t, e: e, d: d, c: conn, br: bufio.NewReaderSize(conn, 1<<17)}
dc.session = e.session(d, nil)
dc.send("HELLO 2 " + dc.session)
m, perr := parseLine(dc.recv(), dirN2D)
if perr != nil || m.str("verb") != "CHALLENGE" {
t.Fatal("no challenge")
}
dc.send("PROOF " + e.proof(d, dc.session, m.str("nonce"), nil))
if line := dc.recv(); !strings.HasPrefix(line, "OK ") {
t.Fatalf("%q", line)
}
// the renewal lands on disk, the node is not restarted
time.Sleep(20 * time.Millisecond)
files.fresh(t, 2, testNodeHost)
if got := servedSerial(t, e.httpsAddr); got != 2 {
t.Fatalf("after the renewal the node served %d", got)
}
// the connection that was up goes on
dc.send("PING")
if line := dc.recv(); line != "PONG" {
t.Fatalf("a connection made with the old certificate: %q", line)
}
if !e.srv.Online(d.host) {
t.Fatal("the device was dropped by the renewal")
}
if want := r.NotAfter(); want.Before(time.Now().Add(50 * 24 * time.Hour)) {
t.Fatalf("not_after %s", want)
}
}
func TestCertReload_ARenewalThatIsNotGoodNeverReplacesTheOneThatWorks(t *testing.T) {
files := newCertFiles(t)
files.fresh(t, 1, testNodeHost)
var logs syncBuf
r, err := newCertReloader(files.cert, files.key, testNodeHost, clock{}, slog.New(slog.NewJSONHandler(&logs, nil)))
if err != nil {
t.Fatal(err)
}
r.lookGap = 0
e := certNode(t, r)
good := func() {
t.Helper()
if got := servedSerial(t, e.httpsAddr); got != 1 {
t.Fatalf("the node serves %d, the certificate that works is 1", got)
}
}
now := time.Now()
for name, write := range map[string]func(){
"another name": func() { files.write(t, 10, now.Add(-time.Hour), now.Add(48*time.Hour), "someone-else.example.test") },
"expired": func() { files.write(t, 11, now.Add(-72*time.Hour), now.Add(-time.Hour), testNodeHost) },
"not yet valid": func() { files.write(t, 12, now.Add(48*time.Hour), now.Add(96*time.Hour), testNodeHost) },
"garbage": func() { files.put(t, []byte("garbage"), []byte("garbage")) },
"half written": func() { _ = os.WriteFile(files.cert, []byte("-----BEGIN CERTIFICATE-----\nAAAA"), 0o600) },
"a missing key": func() { _ = os.Remove(files.key) },
"a missing cert": func() { files.fresh(t, 13, testNodeHost); _ = os.Remove(files.cert) },
"a certificate and a key that are not a pair": func() {
other := newCertFiles(t)
other.fresh(t, 14, testNodeHost)
certPEM, _ := os.ReadFile(other.cert)
third := newCertFiles(t)
third.fresh(t, 15, testNodeHost)
keyPEM, _ := os.ReadFile(third.key)
files.put(t, certPEM, keyPEM)
},
} {
time.Sleep(15 * time.Millisecond)
write()
good()
if name != "a missing key" && name != "a missing cert" && name != "half written" && r.LastError() == "" {
t.Errorf("%s: the refusal is not recorded", name)
}
}
if !strings.Contains(logs.String(), "node_cert_reload_refused") {
t.Fatal("refusals are not logged")
}
// the files are in a bad state now; a good renewal still gets in
time.Sleep(20 * time.Millisecond)
files.fresh(t, 3, testNodeHost)
if got := servedSerial(t, e.httpsAddr); got != 3 {
t.Fatalf("a good renewal after the bad ones: served %d", got)
}
if r.LastError() != "" {
t.Fatalf("last error %q", r.LastError())
}
}
func TestCertReload_ARefusedRenewalIsNotParsedAgainAtEveryHandshake(t *testing.T) {
files := newCertFiles(t)
files.fresh(t, 1, testNodeHost)
var logs syncBuf
r, err := newCertReloader(files.cert, files.key, testNodeHost, clock{}, slog.New(slog.NewJSONHandler(&logs, nil)))
if err != nil {
t.Fatal(err)
}
r.lookGap = 0
time.Sleep(15 * time.Millisecond)
files.put(t, []byte("garbage"), []byte("garbage"))
for i := 0; i < 50; i++ {
if pair, err := r.get(nil); err != nil || pair.Leaf.SerialNumber.Int64() != 1 {
t.Fatalf("handshake %d: %v", i, err)
}
}
if n := logs.Count("node_cert_reload_refused"); n != 1 {
t.Fatalf("one bad renewal was parsed and refused %d times", n)
}
}
func TestCertReload_ReloadNowIsWhatSIGHUPDoes(t *testing.T) {
files := newCertFiles(t)
files.fresh(t, 1, testNodeHost)
r, err := newCertReloader(files.cert, files.key, testNodeHost, clock{}, nil)
if err != nil {
t.Fatal(err)
}
// a look gap: the handshake path has just looked and does not look again for a while, SIGHUP does not wait
_, _ = r.get(nil)
files.fresh(t, 2, testNodeHost)
if pair, _ := r.get(nil); pair.Leaf.SerialNumber.Int64() != 1 {
t.Fatal("looked inside the gap")
}
if err := r.Reload(); err != nil {
t.Fatal(err)
}
if pair, _ := r.get(nil); pair.Leaf.SerialNumber.Int64() != 2 {
t.Fatal("Reload did not put the new certificate in service")
}
files.write(t, 3, time.Now().Add(-time.Hour), time.Now().Add(-time.Minute), testNodeHost)
if err := r.Reload(); err == nil || r.LastError() == "" {
t.Fatal("an expired certificate was accepted by Reload")
}
if pair, _ := r.get(nil); pair.Leaf.SerialNumber.Int64() != 2 {
t.Fatal("a refused Reload replaced the certificate")
}
}
func TestCertReload_TheWatcherPicksUpARenewalWithoutAnyHandshakeAndWarnsBeforeTheEnd(t *testing.T) {
files := newCertFiles(t)
files.write(t, 1, time.Now().Add(-time.Hour), time.Now().Add(3*24*time.Hour), testNodeHost) // 3 days left: under the warning
var logs syncBuf
clk := newFakeClock()
r, err := newCertReloader(files.cert, files.key, testNodeHost, newClock(clk.Now, clk.AfterFunc), slog.New(slog.NewJSONHandler(&logs, nil)))
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() { r.Run(ctx); close(done) }()
eventually(t, "the first look and its warning", 2*time.Second, func() bool { return logs.Count("node_cert_expires_soon") == 1 })
eventually(t, "the watcher sleeping", 2*time.Second, func() bool { return clk.pending() == 1 })
time.Sleep(20 * time.Millisecond)
files.fresh(t, 2, testNodeHost)
clk.Advance(certWatchEvery)
eventually(t, "the renewal in service", 2*time.Second, func() bool { return logs.Count("node_cert_reloaded") == 1 })
if pair, _ := r.get(nil); pair.Leaf.SerialNumber.Int64() != 2 {
t.Fatal("not in service")
}
eventually(t, "the watcher sleeping again", 2*time.Second, func() bool { return clk.pending() == 1 })
clk.Advance(certWatchEvery)
time.Sleep(50 * time.Millisecond)
if logs.Count("node_cert_expires_soon") != 1 {
t.Fatal("warned about a certificate with 60 days left")
}
cancel()
select {
case <-done:
case <-time.After(2 * time.Second):
t.Fatal("the watcher did not stop")
}
}
func TestCertReload_TheNodeDoesNotStartWithoutAUsableCertificate(t *testing.T) {
files := newCertFiles(t)
if _, err := newCertReloader(files.cert, files.key, testNodeHost, clock{}, nil); err == nil {
t.Fatal("started without files")
}
files.write(t, 1, time.Now().Add(-time.Hour), time.Now().Add(time.Hour), "other.example.test")
if _, err := newCertReloader(files.cert, files.key, testNodeHost, clock{}, nil); err == nil || !strings.Contains(err.Error(), testNodeHost) {
t.Fatalf("started with a certificate for another name: %v", err)
}
files.write(t, 2, time.Now().Add(-48*time.Hour), time.Now().Add(-time.Hour), testNodeHost)
if _, err := newCertReloader(files.cert, files.key, testNodeHost, clock{}, nil); err == nil {
t.Fatal("started with an expired certificate")
}
}
files/tunnel/clock.go 67 lines · 2.1 KB
package main
import (
"context"
"math/rand/v2"
"time"
)
// clock is the node's source of time. Production uses the real one; the tests that cover hours (an outage of two hours, a grace of fifteen minutes,
// a JWKS that goes stale) drive their own by hand, so nothing in the node reads the wall clock for a decision except through it. Socket deadlines are
// not decisions of that kind: they stay on the real clock.
type clock struct {
now func() time.Time
afterFunc func(d time.Duration, f func()) (stop func() bool)
}
func newClock(now func() time.Time, afterFunc func(d time.Duration, f func()) func() bool) clock {
if now == nil {
now = time.Now
}
if afterFunc == nil {
afterFunc = func(d time.Duration, f func()) func() bool { return time.AfterFunc(d, f).Stop }
}
return clock{now: now, afterFunc: afterFunc}
}
func (c clock) Now() time.Time { return c.now() }
// AfterFunc runs f once, d from now on this clock, in its own goroutine. The returned function stops it and reports whether it did so in time.
func (c clock) AfterFunc(d time.Duration, f func()) (stop func() bool) {
return c.afterFunc(max(d, 0), f)
}
// sleep waits d on this clock; it reports false when ctx ended first.
func (c clock) sleep(ctx context.Context, d time.Duration) bool {
if ctx.Err() != nil {
return false
}
done := make(chan struct{})
stop := c.AfterFunc(d, func() { close(done) })
defer stop()
select {
case <-done:
return true
case <-ctx.Done():
return false
}
}
// jitter spreads d by up to +/- pct percent, so that a fleet of nodes or a loop restarted together does not stay in step.
func jitter(d time.Duration, pct int) time.Duration {
if pct <= 0 || d <= 0 {
return d
}
span := int64(d) * int64(pct) / 100
return d + time.Duration(rand.Int64N(2*span+1)-span)
}
// backoff is the delay after the n-th failure in a row (n from 1): the policy's minimum, doubled each time, up to its maximum.
func backoff(n int) time.Duration {
d := secs(policyData.Timing.BackoffMinS)
limit := secs(policyData.Timing.BackoffMaxS)
for i := 1; i < n && d < limit; i++ {
d *= 2
}
return min(d, limit)
}
files/tunnel/clock_test.go 200 lines · 6.0 KB
package main
import (
"context"
"errors"
"sort"
"sync"
"testing"
"time"
)
// fakeClock is a clock the tests move by hand: Advance sets the time forward and runs every timer that came due, each in its own goroutine (as
// time.AfterFunc does), in the order of their moments. A timer asked for a moment that is already past runs at once. Nothing else in the node reads
// it: socket deadlines stay on the real clock.
type fakeClock struct {
mu sync.Mutex
now time.Time
timers []*fakeTimer
fired int // timers run so far (a test waits for them)
}
type fakeTimer struct {
when time.Time
f func()
stopped bool
done bool
}
func newFakeClock() *fakeClock { return &fakeClock{now: time.Now()} }
func (c *fakeClock) Now() time.Time {
c.mu.Lock()
defer c.mu.Unlock()
return c.now
}
func (c *fakeClock) AfterFunc(d time.Duration, f func()) func() bool {
c.mu.Lock()
t := &fakeTimer{when: c.now.Add(d), f: f}
if d <= 0 {
t.done = true
c.fired++
c.mu.Unlock()
go f()
return func() bool { return false }
}
c.timers = append(c.timers, t)
c.mu.Unlock()
return func() bool {
c.mu.Lock()
defer c.mu.Unlock()
was := !t.stopped && !t.done
t.stopped = true
return was
}
}
// Advance moves the clock forward by d and runs what came due.
func (c *fakeClock) Advance(d time.Duration) {
c.mu.Lock()
c.now = c.now.Add(d)
var due []*fakeTimer
rest := c.timers[:0]
for _, t := range c.timers {
switch {
case t.stopped || t.done:
case !t.when.After(c.now):
t.done = true
c.fired++
due = append(due, t)
default:
rest = append(rest, t)
}
}
c.timers = rest
c.mu.Unlock()
sort.SliceStable(due, func(i, j int) bool { return due[i].when.Before(due[j].when) })
for _, t := range due {
go t.f()
}
}
// pending is the number of timers waiting.
func (c *fakeClock) pending() int {
c.mu.Lock()
defer c.mu.Unlock()
n := 0
for _, t := range c.timers {
if !t.stopped && !t.done {
n++
}
}
return n
}
// ---------------------------------------------------------------------------------------------------------------------------- the helpers of clock.go
func TestClock_TheFakeClockRunsDueTimersInOrderStopsOnRequestAndRunsAPastOneAtOnce(t *testing.T) {
c := newFakeClock()
var mu sync.Mutex
var order []string
note := func(s string) func() { return func() { mu.Lock(); order = append(order, s); mu.Unlock() } }
c.AfterFunc(20*time.Second, note("b"))
c.AfterFunc(10*time.Second, note("a"))
stopped := c.AfterFunc(15*time.Second, note("stopped"))
c.AfterFunc(40*time.Second, note("later"))
if !stopped() {
t.Fatal("a pending timer was not stopped")
}
if stopped() {
t.Fatal("a timer was stopped twice")
}
c.Advance(25 * time.Second)
eventually(t, "the two due timers", time.Second, func() bool { mu.Lock(); defer mu.Unlock(); return len(order) == 2 })
if c.pending() != 1 {
t.Fatalf("pending %d", c.pending())
}
c.AfterFunc(-time.Second, note("past")) // a moment already gone runs at once
eventually(t, "a past timer", time.Second, func() bool { mu.Lock(); defer mu.Unlock(); return len(order) == 3 })
mu.Lock()
defer mu.Unlock()
if order[0] != "a" && order[0] != "b" || order[2] != "past" {
t.Fatalf("order %v", order)
}
}
func TestClock_SleepEndsWithTheClockOrWithTheContext(t *testing.T) {
c := newFakeClock()
clk := newClock(c.Now, c.AfterFunc)
done := make(chan bool, 1)
go func() { done <- clk.sleep(context.Background(), time.Minute) }()
eventually(t, "the sleeper", time.Second, func() bool { return c.pending() == 1 })
c.Advance(time.Minute)
if !<-done {
t.Fatal("a sleep that ran to its end reported that it was cut")
}
ctx, cancel := context.WithCancel(context.Background())
go func() { done <- clk.sleep(ctx, time.Hour) }()
eventually(t, "the sleeper", time.Second, func() bool { return c.pending() == 1 })
cancel()
if <-done {
t.Fatal("a cancelled sleep reported that it ran to its end")
}
if c.pending() != 0 {
t.Fatal("a cancelled sleep left its timer behind")
}
if clk.sleep(ctx, time.Second) {
t.Fatal("a sleep started with a cancelled context")
}
}
func TestClock_BackoffDoublesFromThePolicysMinimumToItsMaximumAndJitterStaysInItsBand(t *testing.T) {
want := []time.Duration{1, 2, 4, 8, 16, 32, 60, 60}
for i, w := range want {
if got := backoff(i + 1); got != w*time.Second {
t.Errorf("backoff(%d) = %s, want %s", i+1, got, w*time.Second)
}
}
if backoff(0) != secs(policyData.Timing.BackoffMinS) || backoff(1000) != secs(policyData.Timing.BackoffMaxS) {
t.Fatal("backoff out of its bounds")
}
for i := 0; i < 500; i++ {
if got := jitter(100*time.Second, 10); got < 90*time.Second || got > 110*time.Second {
t.Fatalf("jitter %s", got)
}
}
if jitter(time.Second, 0) != time.Second || jitter(0, 10) != 0 {
t.Fatal("jitter of nothing")
}
}
func TestHealth_AnAnswerOfAnyKindIsReachabilityAndDownNeedsAFailureAfterItAndQuiet(t *testing.T) {
var h cpHealth
now := time.Now()
if h.down(now, time.Second) {
t.Fatal("down before anything was seen")
}
h.record(now, nil)
h.record(now.Add(time.Second), &ControlError{Status: 402, Code: codeLicenseInactive}) // a refusal is an answer
h.record(now.Add(2*time.Second), &ControlError{Status: 404, Code: "not_enrolled"})
if h.down(now.Add(time.Hour), time.Second) {
t.Fatal("a control plane that said no is down")
}
for name, err := range map[string]error{"a 503": &ControlError{Status: 503, Code: "end_to_end_unavailable"}, "a 429": &ControlError{Status: 429, Code: "busy"},
"a timeout": context.DeadlineExceeded, "a transport error": errors.New("connection refused"), "an unreadable answer": &ControlError{Status: 502, Code: "bad_response"}} {
var g cpHealth
g.record(now, nil)
g.record(now.Add(time.Second), err)
if g.down(now.Add(2*time.Second), time.Minute) {
t.Errorf("%s: down before the quiet period", name)
}
if !g.down(now.Add(2*time.Minute), time.Minute) {
t.Errorf("%s: not down after the quiet period", name)
}
g.record(now.Add(3*time.Minute), nil)
if g.down(now.Add(10*time.Minute), time.Minute) {
t.Errorf("%s: still down after an answer", name)
}
}
}
files/tunnel/config.go 263 lines · 15.2 KB
package main
import (
"context"
"crypto/tls"
"fmt"
"io"
"log/slog"
"net"
"net/netip"
"net/url"
"time"
)
// Config is everything the node needs. DefaultConfig gives the production values, every one of them read from policy.json; tests override the
// timeouts and addresses.
type Config struct {
HTTPSAddr string // :443 the one public listener: own-name SNI is the control handler (ALPN all-tunnel/2), any other SNI is spliced. "" disables.
HTTPAddr string // :80 only the node's own ACME webroot. "" disables.
NodeID string // the node's id in the control plane (authorize carries it; OK says it)
NodeHost string // the node's own name: the SNI that reaches the control handler, and the host of the proof's URL
PublicPort int // the port devices dial, only for the proof's URL (443)
ACMEWebroot string // directory whose .well-known/acme-challenge the node's own certificate client writes; "" serves nothing on :80
// CertSource gives the node's public-CA certificate for its own name (hot reload is the certificate piece's business, the node asks at every
// handshake).
CertSource func(*tls.ClientHelloInfo) (*tls.Certificate, error)
// Keys are the control plane's public keys: sessions are verified offline against them.
Keys KeySource
// Control is the node's side of POST /v1/tunnel/authorize.
Control ControlPlane
// Feed is the node's side of GET /v1/tunnel/feed: what the node must stop serving (a revoke, a pause, a released label, the kill switch). Nil runs
// the node without a feed (the test node's fake control plane has none).
Feed FeedSource
// Health is the node's record of whether its control plane answers (the paid grace asks it); the JWKS cache and the feed loop write to the same
// one. Nil gets a fresh one.
Health *cpHealth
// Background are tasks that live as long as the node: the JWKS refresh, the certificate watcher. They start with the listeners and end when the
// node shuts down.
Background []func(ctx context.Context)
MaxStreams int // concurrent client connections, global
MaxStreamsPerHost int // concurrent client connections per device host
MaxUnauth int // concurrent connections that have not yet finished their first message (a ClientHello or a first line)
MaxStreamBytes int64 // bytes (both directions summed) per stream
ACMEMaxBytes int64 // bytes of a webroot challenge file served on :80
HelloTimeout time.Duration // a client must deliver a full ClientHello within this
AttachTimeout time.Duration // a device must ATTACH within this after OPEN
FirstLineTimeout time.Duration // TLS handshake + first protocol line; also the time a device has to answer CHALLENGE with PROOF
DeadPeer time.Duration // a control connection that says nothing for this long is dropped
MinRenewGap time.Duration // a RENEW sooner than this after the previous one is busy: a device cannot turn one connection into a stream of authorize calls
WriteTimeout time.Duration // one control-line write
StreamIdle time.Duration // no bytes in a direction for this long closes the stream
StreamMaxAge time.Duration // hard cap on one stream's lifetime
HalfCloseTimeout time.Duration // after one direction ends cleanly, the other has this long
ACMETimeout time.Duration // a whole request on :80
AuthorizeMax int // calls to authorize in flight at once, whoever asks
AuthorizeTimeout time.Duration // the control plane's time to answer authorize; beyond it the device gets control_unavailable
ShutdownGrace time.Duration
FeedPoll time.Duration // the longest the node waits between two looks at the feed, and the longest a poll asks the control plane to hold an answer
FeedStartWait time.Duration // how long Start waits for the feed's first answer before it accepts connections (the node starts without it when it is late)
Grace time.Duration // how long a paid connection outlives its valid_until while the control plane does not answer
DownQuiet time.Duration // how long nothing may have answered for the control plane to count as down
DrainQuiet time.Duration // how long a draining node keeps its open streams before it cuts them
// Hardening (limits.go, hints.go, gated.go, openai_prefixes.go, metrics.go). Every number comes from policy.json (node_limits, gated).
PerIPConcurrent int // connections one source holds at once: pre-hello, and the client streams it has open (an IPv6 source is its /64)
PerIPNewPerS int // new connections per second one source may open (a burst of this many, then this many per second)
IPv6PrefixBits int // the length of the IPv6 prefix that counts as one source
IPTableMax int // sources remembered at once; beyond it new ones share one overflow entry
OpenAIPerIPFactor int // the per-source limits of an address in the OpenAI prefix list are this many times larger (one egress address serves many users)
DevicePerIPFactor int // the same for an address that holds an authenticated control connection: a device opens one connection per stream it serves
UnknownSNIPerIP int // distinct names that are not a device online (or malformed ClientHellos) one source may try within UnknownSNIWindow
UnknownSNIWindow time.Duration //
UnknownSNIBan time.Duration // how long a source that went over the budget is dropped at accept (memory only)
LabelAnonSlice int // of MaxStreamsPerHost, what a source that is not marked good may hold; the rest is the slice of sources the device marked with HINT
LabelNewPerS int // new client streams per second one device host accepts from sources that are not marked good
LabelNewBurst int //
OpenAIReservedPct int // of MaxStreams, the share only sources of the openai or hint class may use
HintTTL time.Duration // how long a HINT keeps an address good
HintsPerLabel int // addresses remembered per device host
GatedBytesPerDay int64 // a gated device's byte budget per UTC day (both directions, all its streams)
GatedStreamsPerDay int // and its stream budget
TCPKeepalive time.Duration // keepalive period of every socket the node holds
MetricsAddr string // loopback address of /metrics and /healthz; "" disables
OpenAIPrefixesURL string // where the connector prefixes are published; "" disables the refresh
OpenAIPrefixesCache string // file with the last good list (public data), read at start and written at every good refresh; "" keeps it in memory only
OpenAIPrefixRefresh time.Duration
// PrefixFetch replaces the HTTPS GET of OpenAIPrefixesURL (tests, and a deployment that mirrors the list).
PrefixFetch func(ctx context.Context) ([]byte, error)
// CertNotAfter reports the end of the node's own certificate, for the metrics (nil: not reported).
CertNotAfter func() time.Time
// ClockSkew is how far a device's clock may be ahead of the node's: a session or proof issued up to this far in the future still verifies. The
// device compensates its clock with the control plane's server_time, this is the window left over.
ClockSkew time.Duration
Logger *slog.Logger
// Now is the node's clock (time.Now when nil), AfterFunc its timers (time.AfterFunc when nil). Every lifetime the node decides (valid_until, the
// grace, the feed's pace, a drain's quiet period, the JWKS cache) runs on them, so a test drives hours in an instant.
Now func() time.Time
AfterFunc func(d time.Duration, f func()) (stop func() bool)
// Classify overrides the class of a client's source address for the OPEN: openai, hint or anon. Nil is the node's own: openai for an address of
// the OpenAI prefix list, hint for one the device marked with HINT, anon for the rest.
Classify func(peer netip.Addr, host string) string
// OnHint is told of the source addresses a device marked as good with HINT, besides the node's own hint table (nil: only the table).
OnHint func(host string, addrs []netip.Addr)
// tap, when non-nil, is called with every payload buffer the node forwards (never set in production). Tests use it to prove what the node
// can and cannot see.
tap func(streamID, direction string, p []byte)
// divert, when non-nil, may take a client connection before the node opens a stream for it (never set in production: only the test-only mutation of
// blindmutant_test_hook.go sets it, to be a node that does what the blindness gate must catch). It reports whether it handled the connection.
divert func(c net.Conn, peeked []byte, log *slog.Logger) bool
// peerOf, when non-nil, tells the source address of an accepted connection instead of its remote address (never set in production: a test gives
// loopback connections the addresses of many sources).
peerOf func(c net.Conn) (netip.Addr, bool)
// trace, when non-nil, receives one event per protocol step in the format of the contracts' transcript checker (never set in production).
trace func(event map[string]any)
}
// DefaultConfig returns the production defaults: every window and limit comes from policy.json.
func DefaultConfig() Config {
p := policyData
return Config{
HTTPSAddr: ":443",
HTTPAddr: ":80",
PublicPort: 443,
MaxStreams: p.NodeLimits.GlobalStreamsMax,
MaxStreamsPerHost: p.NodeLimits.LabelStreamsMax,
MaxUnauth: p.NodeLimits.MaxUnauthenticatedControl,
MaxStreamBytes: 512 << 20,
ACMEMaxBytes: 8 << 10,
HelloTimeout: secs(p.NodeLimits.ClientHelloDeadlineS),
AttachTimeout: secs(p.Protocol.AttachTimeoutS),
FirstLineTimeout: secs(p.Protocol.FirstLineTimeoutS),
DeadPeer: secs(p.Timing.DeadPeerS),
MinRenewGap: secs(p.Timing.BackoffMinS),
WriteTimeout: 5 * time.Second,
StreamIdle: secs(p.Timing.StreamIdleS),
StreamMaxAge: secs(p.Timing.StreamMaxAgeS),
HalfCloseTimeout: secs(p.Timing.HalfCloseS),
ACMETimeout: 15 * time.Second,
AuthorizeMax: p.Authorization.AuthorizeMaxConcur,
AuthorizeTimeout: 10 * time.Second,
ShutdownGrace: 10 * time.Second,
FeedPoll: secs(p.Timing.FeedPollS),
FeedStartWait: 3 * time.Second,
Grace: secs(p.Authorization.GracePaidS),
DownQuiet: 2 * secs(p.Timing.FeedPollS),
DrainQuiet: secs(p.Timing.DrainQuietS),
ClockSkew: secs(p.Authorization.NodeSkewS),
PerIPConcurrent: p.NodeLimits.PerIPConcurrent,
PerIPNewPerS: p.NodeLimits.PerIPNewPerS,
IPv6PrefixBits: p.NodeLimits.IPv6PrefixBits,
IPTableMax: p.NodeLimits.IPTableMax,
OpenAIPerIPFactor: p.NodeLimits.OpenAIPerIPFactor,
DevicePerIPFactor: p.NodeLimits.DevicePerIPFactor,
UnknownSNIPerIP: p.NodeLimits.UnknownSNIPerIP,
UnknownSNIWindow: secs(p.NodeLimits.UnknownSNIWindowS),
UnknownSNIBan: secs(p.NodeLimits.UnknownSNIBanS),
LabelAnonSlice: p.NodeLimits.LabelAnonSlice,
LabelNewPerS: p.NodeLimits.LabelNewPerS,
LabelNewBurst: p.NodeLimits.LabelNewBurst,
OpenAIReservedPct: p.NodeLimits.OpenAIReservedPct,
HintTTL: secs(p.NodeLimits.HintTTLS),
HintsPerLabel: p.NodeLimits.HintAddressesPerLabel,
GatedBytesPerDay: p.Gated.BytesPerDay,
GatedStreamsPerDay: p.Gated.StreamsPerDay,
TCPKeepalive: secs(p.NodeLimits.TCPKeepaliveS),
OpenAIPrefixRefresh: secs(p.NodeLimits.OpenAIPrefixRefreshS),
Logger: slog.New(slog.NewJSONHandler(io.Discard, nil)),
}
}
// validate refuses a configuration the node cannot run safely. It does not touch the network.
func (c *Config) validate() error {
if !reNodeID.MatchString(c.NodeID) {
return fmt.Errorf("node id %q: want 1-16 characters of [a-z0-9]", c.NodeID)
}
if c.HTTPSAddr != "" {
host, ok := normalizeName(c.NodeHost)
if !ok || host != c.NodeHost {
return fmt.Errorf("node host %q is not a normalized DNS name", c.NodeHost)
}
if validFQDN(host) {
return fmt.Errorf("node host %q looks like a device host", host)
}
if c.CertSource == nil {
return fmt.Errorf("no certificate source for the node's own name")
}
}
if c.Keys == nil || c.Control == nil {
return fmt.Errorf("the node needs a key source and a control plane")
}
if c.PublicPort < 1 || c.PublicPort > 65535 {
return fmt.Errorf("public port %d", c.PublicPort)
}
if c.ClockSkew < 0 || c.ClockSkew > secs(policyData.Authorization.NodeSkewMaxS) {
return fmt.Errorf("clock skew %s is beyond the policy's %s", c.ClockSkew, secs(policyData.Authorization.NodeSkewMaxS))
}
if c.Feed != nil && (c.FeedPoll < time.Millisecond || c.FeedPoll > secs(policyData.Timing.FeedPollS)) {
return fmt.Errorf("feed poll %s is beyond the policy's %s", c.FeedPoll, secs(policyData.Timing.FeedPollS))
}
if c.DrainQuiet < 0 || c.DrainQuiet > 24*time.Hour {
return fmt.Errorf("drain quiet period %s: want 0 to 24h", c.DrainQuiet)
}
if c.Grace < 0 || c.Grace > secs(policyData.Authorization.GracePaidS) {
return fmt.Errorf("grace %s is beyond the policy's %s", c.Grace, secs(policyData.Authorization.GracePaidS))
}
if c.AuthorizeMax < 1 {
return fmt.Errorf("authorize concurrency must be at least 1")
}
if c.MaxStreams < 1 || c.MaxStreamsPerHost < 1 || c.MaxUnauth < 1 {
return fmt.Errorf("stream limits must be at least 1")
}
if c.PerIPConcurrent < 1 || c.PerIPNewPerS < 1 || c.IPTableMax < 1 || c.OpenAIPerIPFactor < 1 || c.DevicePerIPFactor < 1 || c.LabelAnonSlice < 1 || c.LabelNewPerS < 1 || c.LabelNewBurst < 1 {
return fmt.Errorf("per-source and per-device limits must be at least 1")
}
if c.IPv6PrefixBits < 1 || c.IPv6PrefixBits > 128 {
return fmt.Errorf("ipv6 prefix of %d bits", c.IPv6PrefixBits)
}
if c.OpenAIReservedPct < 0 || c.OpenAIReservedPct > 90 {
return fmt.Errorf("openai reserved share %d%%: want 0 to 90", c.OpenAIReservedPct)
}
if c.UnknownSNIPerIP < 1 || c.UnknownSNIWindow <= 0 || c.UnknownSNIBan <= 0 || c.HintTTL <= 0 || c.HintsPerLabel < 1 || c.TCPKeepalive <= 0 {
return fmt.Errorf("the unknown-name budget, the hint table and the keepalive need positive values")
}
if c.GatedBytesPerDay < 1 || c.GatedStreamsPerDay < 1 {
return fmt.Errorf("a gated device's daily budgets must be at least 1")
}
if c.MetricsAddr != "" {
if err := checkLoopbackAddr(c.MetricsAddr); err != nil {
return err
}
}
if c.OpenAIPrefixesURL != "" {
if u, err := url.Parse(c.OpenAIPrefixesURL); err != nil || u.Host == "" || u.User != nil || (u.Scheme != "https" && !(u.Scheme == "http" && isLoopbackHost(u.Hostname()))) {
return fmt.Errorf("prefix list URL %q: want https://<host>/<path> (plain http only for a loopback mirror)", c.OpenAIPrefixesURL)
}
}
return nil
}
// checkLoopbackAddr refuses a metrics address that is not loopback: the counters are for the machine's own operator, never for the network.
func checkLoopbackAddr(addr string) error {
host, _, err := net.SplitHostPort(addr)
if err != nil {
return fmt.Errorf("metrics address %q: %w", addr, err)
}
if !isLoopbackHost(host) {
return fmt.Errorf("metrics address %q is not a loopback address: /metrics and /healthz are for this machine only", addr)
}
return nil
}
// isLoopbackHost reports whether host is a loopback IP literal.
func isLoopbackHost(host string) bool {
ip, err := netip.ParseAddr(host)
return err == nil && ip.IsLoopback()
}
files/tunnel/config_test.go 410 lines · 15.5 KB
package main
import (
"bufio"
"bytes"
"crypto/ed25519"
"crypto/rand"
"crypto/tls"
"crypto/x509"
"encoding/json"
"io"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"sync"
"syscall"
"testing"
"time"
)
func TestConfig_RefusesWhatTheNodeCannotRunSafely(t *testing.T) {
e := newTestEnv(t, nil)
good := func() Config {
cfg := DefaultConfig()
cfg.NodeID, cfg.NodeHost, cfg.Keys, cfg.Control = testNodeID, testNodeHost, e.keys, e.cp
cfg.CertSource = func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return nil, nil }
return cfg
}
cfg := good()
if _, err := NewServer(cfg); err != nil {
t.Fatalf("the baseline configuration: %v", err)
}
for name, mutate := range map[string]func(*Config){
"no node id": func(c *Config) { c.NodeID = "" },
"a node id with a capital": func(c *Config) { c.NodeID = "T1" },
"a node id of 17 characters": func(c *Config) { c.NodeID = strings.Repeat("a", 17) },
"no node host": func(c *Config) { c.NodeHost = "" },
"a node host with a capital": func(c *Config) { c.NodeHost = "T1.nodes.all-e2e.test" },
"a node host with a trailing dot": func(c *Config) { c.NodeHost = testNodeHost + "." },
"a node host that is an address": func(c *Config) { c.NodeHost = "192.0.2.1" },
"a node host that is a device host": func(c *Config) {
c.NodeHost = label20() + "." + testZone
},
"no certificate source": func(c *Config) { c.CertSource = nil },
"no key source": func(c *Config) { c.Keys = nil },
"no control plane": func(c *Config) { c.Control = nil },
"port zero": func(c *Config) { c.PublicPort = 0 },
"port 70000": func(c *Config) { c.PublicPort = 70000 },
"a negative skew": func(c *Config) { c.ClockSkew = -time.Second },
"no stream limit": func(c *Config) { c.MaxStreams = 0 },
// WP06: nothing the policy bounds may be configured beyond it
"a grace longer than the policy's": func(c *Config) { c.Grace = secs(policyData.Authorization.GracePaidS) + time.Second },
"a negative grace": func(c *Config) { c.Grace = -time.Second },
"a feed polled less often than the policy": func(c *Config) { c.Feed, c.FeedPoll = &scriptedFeed{}, secs(policyData.Timing.FeedPollS)+time.Second },
"a feed polled in no time": func(c *Config) { c.Feed, c.FeedPoll = &scriptedFeed{}, 0 },
"a negative drain": func(c *Config) { c.DrainQuiet = -time.Second },
"a drain of two days": func(c *Config) { c.DrainQuiet = 48 * time.Hour },
} {
cfg := good()
mutate(&cfg)
if _, err := NewServer(cfg); err == nil {
t.Errorf("%s: accepted", name)
}
}
// a configuration with the public listener off needs no own name or certificate
cfg = good()
cfg.HTTPSAddr, cfg.NodeHost, cfg.CertSource = "", "", nil
if _, err := NewServer(cfg); err != nil {
t.Errorf("no public listener: %v", err)
}
}
func TestRun_TheProductionBuildNeedsItsSigningKeyAndTheOld11FlagsAreGone(t *testing.T) {
var out, errb bytes.Buffer
if rc := run([]string{"-version"}, &out, &errb); rc != 0 || !strings.HasPrefix(out.String(), "all-tunnel ") {
t.Fatalf("-version: %d %q %q", rc, out.String(), errb.String())
}
errb.Reset()
// without its signing key (or, in the test node's build, without its fakes) the node does not start: it never runs without verifying anyone
if rc := run([]string{"-node-id", "t1", "-node-host", testNodeHost}, io.Discard, &errb); rc != 1 ||
(!strings.Contains(errb.String(), "signing key") && !strings.Contains(errb.String(), "needs -test-jwks")) {
t.Fatalf("no backend configured: %d %q", rc, errb.String())
}
if rc := run([]string{"-no-such-flag"}, io.Discard, io.Discard); rc != 2 {
t.Fatalf("unknown flag: %d", rc)
}
if rc := run([]string{"-log-level", "loud"}, io.Discard, io.Discard); rc != 2 {
t.Fatalf("bad log level: %d", rc)
}
for _, old := range []string{"-control-addr", "-state-dir", "-token-file", "-devices-file", "-host", "-gen-token"} {
if rc := run([]string{old, "x"}, io.Discard, io.Discard); rc != 2 {
t.Errorf("the 1.1 flag %s is still accepted (exit %d)", old, rc)
}
}
}
func TestFileCert_ReloadsWhenTheFilesChangeAndKeepsTheOneThatWorks(t *testing.T) {
dir := t.TempDir()
write := func(name string) {
_, cert := makeCA(t, name)
certPEM, keyPEM := pemOf(t, cert)
if err := os.WriteFile(filepath.Join(dir, "c.pem"), certPEM, 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "k.pem"), keyPEM, 0o600); err != nil {
t.Fatal(err)
}
}
write("one.example.test")
fc, err := newCertReloader(filepath.Join(dir, "c.pem"), filepath.Join(dir, "k.pem"), "", clock{}, nil)
if err != nil {
t.Fatal(err)
}
fc.lookGap = 0 // the files are looked at on every call, so the test does not wait for the gap
first, _ := fc.get(nil)
again, _ := fc.get(nil)
if first != again {
t.Fatal("an unchanged file was parsed again")
}
time.Sleep(20 * time.Millisecond)
write("two.example.test")
second, err := fc.get(nil)
if err != nil || fingerprintOf(second.Certificate[0]) == fingerprintOf(first.Certificate[0]) {
t.Fatalf("a renewed certificate was not picked up: %v", err)
}
// a half-written renewal does not take the node down: the working certificate stays
if err := os.WriteFile(filepath.Join(dir, "c.pem"), []byte("garbage"), 0o600); err != nil {
t.Fatal(err)
}
kept, err := fc.get(nil)
if err != nil || kept != second {
t.Fatalf("a failed reload must keep the certificate that works: %v", err)
}
if _, err := newCertReloader(filepath.Join(dir, "missing"), filepath.Join(dir, "k.pem"), "", clock{}, nil); err == nil {
t.Fatal("a node cannot start without its certificate")
}
}
// ------------------------------------------------------------------------------------------------------------------------------------
// The test-node binary (ALL_TEST_NODE_BIN)
// ------------------------------------------------------------------------------------------------------------------------------------
var (
testNodeOnce sync.Once
testNodePath string
testNodeErr error
)
// testNodeBinary is the binary the device's tests run against: ALL_TEST_NODE_BIN when set, otherwise this package built with -tags testnode.
func testNodeBinary(t *testing.T) string {
t.Helper()
if p := os.Getenv("ALL_TEST_NODE_BIN"); p != "" {
return p
}
testNodeOnce.Do(func() {
goBin, err := exec.LookPath("go")
if err != nil {
if _, serr := os.Stat("/opt/homebrew/bin/go"); serr != nil {
testNodeErr = err
return
}
goBin = "/opt/homebrew/bin/go"
}
dir, err := os.MkdirTemp("", "all-tunnel-testnode-*")
if err != nil {
testNodeErr = err
return
}
testNodePath = filepath.Join(dir, "all-tunnel-testnode")
out, err := exec.Command(goBin, "build", "-tags", "testnode", "-o", testNodePath, ".").CombinedOutput()
if err != nil {
testNodeErr = &exec.ExitError{Stderr: out}
t.Logf("go build: %s", out)
}
})
if testNodeErr != nil {
t.Skipf("cannot build the test node: %v", testNodeErr)
}
return testNodePath
}
var (
prodBinOnce sync.Once
prodBinPath string
prodBinErr error
)
// productionBinary is this package built the way it is deployed (no test tags): the node a device really meets.
func productionBinary(t *testing.T) string {
t.Helper()
prodBinOnce.Do(func() {
goBin, err := exec.LookPath("go")
if err != nil {
if _, serr := os.Stat("/opt/homebrew/bin/go"); serr != nil {
prodBinErr = err
return
}
goBin = "/opt/homebrew/bin/go"
}
dir, err := os.MkdirTemp("", "all-tunnel-prod-*")
if err != nil {
prodBinErr = err
return
}
prodBinPath = filepath.Join(dir, "all-tunnel")
if out, err := exec.Command(goBin, "build", "-o", prodBinPath, ".").CombinedOutput(); err != nil {
prodBinErr = &exec.ExitError{Stderr: out}
t.Logf("go build: %s", out)
}
})
if prodBinErr != nil {
t.Skipf("cannot build the production node: %v", prodBinErr)
}
return prodBinPath
}
func TestMain(m *testing.M) {
code := m.Run()
if testNodePath != "" {
_ = os.RemoveAll(filepath.Dir(testNodePath))
}
if prodBinPath != "" {
_ = os.RemoveAll(filepath.Dir(prodBinPath))
}
os.Exit(code)
}
func jwksFile(t *testing.T, path, kid string, pub ed25519.PublicKey) {
t.Helper()
doc := map[string]any{"keys": []any{map[string]any{"kty": "OKP", "crv": "Ed25519", "x": b64Encode(pub), "kid": kid, "alg": "EdDSA", "use": "sig", "key_ops": []any{"verify"}}}}
if err := os.WriteFile(path, []byte(canonicalJSON(doc)), 0o600); err != nil {
t.Fatal(err)
}
}
func writeControl(t *testing.T, path string, devices map[string]any) {
t.Helper()
b, _ := json.Marshal(map[string]any{"devices": devices})
tmp := path + ".tmp"
if err := os.WriteFile(tmp, b, 0o600); err != nil {
t.Fatal(err)
}
if err := os.Rename(tmp, path); err != nil {
t.Fatal(err)
}
}
// TestTestNodeBinary_RunsTheRealProtocolAgainstAFileDrivenControlPlane starts the test-node program, plays a device against it and a client of
// the device, changes the fake control plane while it runs and stops it with SIGTERM.
func TestTestNodeBinary_RunsTheRealProtocolAgainstAFileDrivenControlPlane(t *testing.T) {
bin := testNodeBinary(t)
dir := t.TempDir()
pub, priv, _ := ed25519.GenerateKey(rand.Reader)
jwks, control, calls, certOut := filepath.Join(dir, "jwks.json"), filepath.Join(dir, "control.json"), filepath.Join(dir, "calls.jsonl"), filepath.Join(dir, "node.pem")
jwksFile(t, jwks, testKID, pub)
e := &testEnv{t: t, cpKey: priv, tapByID: map[string]*bytes.Buffer{}}
e.srv = &Server{nodeURL: nodeProofURL(testNodeHost, 443)}
d := e.newDeviceNoPlan()
writeControl(t, control, map[string]any{d.id: map[string]any{"host": d.host, "status": "active", "plan": "paid", "valid_for_s": 60}})
cmd := exec.Command(bin, "-https-addr", "127.0.0.1:0", "-http-addr", "127.0.0.1:0", "-node-id", testNodeID, "-node-host", testNodeHost,
"-test-jwks", jwks, "-test-control", control, "-test-calls", calls, "-test-cert-out", certOut, "-log-level", "info")
stdout, _ := cmd.StdoutPipe()
var logs syncBuf
cmd.Stderr = &logs
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
exited := make(chan error, 1)
go func() { exited <- cmd.Wait() }()
t.Cleanup(func() { _ = cmd.Process.Kill() })
ready := make(chan map[string]any, 1)
go func() {
line, _ := bufio.NewReader(stdout).ReadString('\n')
var m map[string]any
if json.Unmarshal([]byte(line), &m) == nil {
ready <- m
}
}()
var info map[string]any
select {
case info = <-ready:
case <-time.After(20 * time.Second):
t.Fatalf("the test node did not announce itself:\n%s", logs.String())
}
if info["ready"] != true || info["node_id"] != testNodeID || info["node_host"] != testNodeHost {
t.Fatalf("ready line %v", info)
}
e.httpsAddr, e.httpAddr = info["https"].(string), info["http"].(string)
certPEM, err := os.ReadFile(certOut)
if err != nil {
t.Fatal(err)
}
e.nodePool = x509.NewCertPool()
if !e.nodePool.AppendCertsFromPEM(certPEM) {
t.Fatal("the node's certificate file is not PEM")
}
dc := e.connectOK(d)
if body := dc.ok["body"].(map[string]any); body["host"] != d.host || body["status"] != "active" {
t.Fatalf("OK %v", body)
}
ds := dc.serve(nil)
c, err := e.clientTLS(d)
if err != nil {
t.Fatal(err)
}
if resp, err := postOver(c, d.host, []byte("via-binary-"+marker)); err != nil || string(resp) != "echo:via-binary-"+marker {
t.Fatalf("resp=%q err=%v\nlogs:\n%s", resp, err, logs.String())
}
_ = c.Close()
// the file is read again at every authorize: this renewal is refused, the next one answers gated
writeControl(t, control, map[string]any{d.id: map[string]any{"host": d.host, "error": map[string]any{"status": 402, "code": "license_inactive"}}})
if r := dc.renew(); r != "ERR license_inactive" {
t.Fatalf("renew with the control plane refusing: %q", r)
}
writeControl(t, control, map[string]any{d.id: map[string]any{"host": d.host, "status": "gated", "plan": "trial", "valid_for_s": 900}})
if r := dc.renew(); r != "ERR busy" { // the production gap between two RENEWs
t.Fatalf("a renew right after another: %q", r)
}
time.Sleep(1200 * time.Millisecond)
if r := dc.renew(); !strings.HasPrefix(r, "OK ") || !strings.Contains(r, `"status":"gated"`) {
t.Fatalf("renew answering gated: %q", r)
}
// a device the file does not know is refused
stranger := e.newDeviceNoPlan()
sc := e.connect(stranger)
if r := sc.hello(func() string { return e.session(stranger, nil) }, func(s, n string) string { return e.proof(stranger, s, n, nil) }); r != "ERR device_revoked" {
t.Fatalf("unknown device: %q", r)
}
// every authorize was logged with its reason
raw, _ := os.ReadFile(calls)
if n := strings.Count(string(raw), `"reason":"renew"`); n != 2 || strings.Count(string(raw), `"reason":"hello"`) != 2 {
t.Fatalf("calls file:\n%s", raw)
}
// ACME path through the binary: only the node's own name, and nothing without a webroot
if out := acmeGet(t, e, "GET", "/.well-known/acme-challenge/anything", testNodeHost); !strings.HasPrefix(out, "HTTP/1.1 404") {
t.Fatalf("acme: %q", out)
}
if err := cmd.Process.Signal(syscall.SIGTERM); err != nil {
t.Fatal(err)
}
select {
case err := <-exited:
if err != nil {
t.Fatalf("exit: %v\nlogs:\n%s", err, logs.String())
}
case <-time.After(20 * time.Second):
t.Fatalf("did not exit on SIGTERM\nlogs:\n%s", logs.String())
}
<-ds.done
// SIGTERM drains (WP06): the device is told to prefer the other node, and the node closes once its streams are gone
if !containsLine(ds.otherLines(), "DRAIN "+strconv.Itoa(policyData.Timing.DrainQuietS)) || !containsLine(ds.otherLines(), "CLOSE drain") {
t.Fatalf("the device was not told the node is draining: %q", ds.otherLines())
}
out := logs.String()
for _, want := range []string{`"msg":"draining"`, `"msg":"drained"`, `"msg":"control_up"`, `"msg":"stream_close"`} {
if !strings.Contains(out, want) {
t.Errorf("missing %s in logs", want)
}
}
for name, secret := range map[string]string{"marker": marker, "host": d.host, "device id": d.id, "session": dc.session} {
if strings.Contains(out, secret) {
t.Fatalf("the process logs name the %s", name)
}
}
}
func TestTestNodeBinary_RefusesToStartWithoutItsFakesOrWithABadConfiguration(t *testing.T) {
bin := testNodeBinary(t)
run := func(args ...string) (string, int) {
cmd := exec.Command(bin, args...)
var out bytes.Buffer
cmd.Stderr, cmd.Stdout = &out, &out
err := cmd.Run()
if ee, ok := err.(*exec.ExitError); ok {
return out.String(), ee.ExitCode()
}
return out.String(), 0
}
dir := t.TempDir()
jwks, control := filepath.Join(dir, "j.json"), filepath.Join(dir, "c.json")
pub, _, _ := ed25519.GenerateKey(rand.Reader)
jwksFile(t, jwks, testKID, pub)
writeControl(t, control, map[string]any{})
base := []string{"-https-addr", "127.0.0.1:0", "-http-addr", "127.0.0.1:0", "-node-id", testNodeID, "-node-host", testNodeHost}
for name, tc := range map[string]struct {
args []string
want string
rc int
}{
"no fakes": {base, "needs -test-jwks and -test-control", 1},
"skew too wide": {append(append([]string{}, base...), "-test-jwks", jwks, "-test-control", control, "-clock-skew", "999"), "clock skew", 1},
"bad node id": {[]string{"-node-id", "T1", "-node-host", testNodeHost, "-test-jwks", jwks, "-test-control", control}, "node id", 1},
"version": {[]string{"-version"}, "all-tunnel", 0},
"unknown flag": {[]string{"-bogus"}, "flag provided but not defined", 2},
"the old 1.1 pin": {[]string{"-control-addr", ":8443"}, "flag provided but not defined", 2},
} {
out, rc := run(tc.args...)
if rc != tc.rc || !strings.Contains(out, tc.want) {
t.Errorf("%s: exit %d, output %q", name, rc, out)
}
}
}
files/tunnel/contracts_test.go 378 lines · 13.8 KB
package main
// ALL 2.0, WP01: the Go side of the contract tests. The golden vectors of contracts/vectors/ are the spec: this file runs every one of them
// through the Go reference verifier (contracts_verify_test.go) and demands the same verdict, and the same error word, as the Python and
// TypeScript verifiers. The node written by WP05 and WP06 is tested against the same functions and files; nothing here is node code.
import (
"os"
"path/filepath"
"reflect"
"strings"
"testing"
)
// ctVectorsDir is contracts/vectors, or the folder ALL_CONTRACT_VECTORS_DIR names (vectors made with another salt: nobody has seen them).
func ctVectorsDir() string {
if dir := os.Getenv("ALL_CONTRACT_VECTORS_DIR"); dir != "" {
return dir
}
return filepath.Join(ctContracts, "vectors")
}
func ctVectors(t *testing.T, name string) map[string]any {
t.Helper()
value, err := ctLoadFile(filepath.Join(ctVectorsDir(), name))
if err != nil {
t.Fatalf("%s: %v", name, err)
}
return value.(map[string]any)
}
func ctList(t *testing.T, doc map[string]any, key string) []map[string]any {
t.Helper()
items, ok := doc[key].([]any)
if !ok || len(items) == 0 {
t.Fatalf("no vectors under %q", key)
}
out := make([]map[string]any, len(items))
for i, item := range items {
out[i] = item.(map[string]any)
}
return out
}
func ctVerdict(r ctResult) map[string]any {
if r.ok {
return map[string]any{"ok": true}
}
return map[string]any{"ok": false, "error": r.err}
}
func ctCheckVerdict(t *testing.T, vector map[string]any, got map[string]any) {
t.Helper()
if !ctEqual(got, vector["expect"]) {
t.Errorf("%s: got %v, want %v", vector["id"], got, vector["expect"])
}
}
func ctStrings(value any) []string {
var out []string
for _, item := range value.([]any) {
out = append(out, item.(string))
}
return out
}
func TestContractsPolicyHoldsThePlansNumbers(t *testing.T) {
want := map[string]int{
"timing.stream_idle_s": 1800, "timing.stream_max_age_s": 14400, "timing.ping_interval_s": 20, "timing.dead_peer_s": 45,
"authorization.valid_until_paid_s": 14400, "authorization.valid_until_trial_s": 900, "authorization.valid_until_gated_s": 900,
"authorization.grace_paid_s": 900, "authorization.access_ttl_paid_s": 14400, "authorization.access_ttl_trial_s": 600,
"timing.drain_quiet_s": 1200, "pairing.window_s": 600, "budget.classes.paid": 20, "budget.classes.trial": 12, "budget.classes.flex": 8,
"gated.bytes_per_day": 10 * 1024 * 1024, "gated.streams_per_day": 200, "gated.cutoff_days": 14, "identity.label_length": 20,
"timing.renew_before_valid_until_s": 300, "jwks.max_keys": 16, "node_limits.label_streams_max": 64, "node_limits.client_hello_deadline_s": 5,
"authorization.session_ttl_s": 300, "authorization.proof_ttl_s": 60, "authorization.node_assertion_ttl_s": 60, "protocol.version": 2,
}
for key, value := range want {
if got := ctPolInt(key); got != value {
t.Errorf("policy %s = %d, want %d", key, got, value)
}
}
if ctPolStr("protocol.alpn") != "all-tunnel/2" {
t.Errorf("alpn = %s", ctPolStr("protocol.alpn"))
}
}
func TestContractsLabelPatternComesFromThePolicyOnly(t *testing.T) {
// the schema, the policy and the compiled verifier pattern are one text; this file never types the pattern itself
label := ctPolStr("identity.label_regex")
defs := ctSchemaSet.docs["common.schema.json"].(map[string]any)["$defs"].(map[string]any)
if defs["label"].(map[string]any)["pattern"] != label || ctLabelRE.String() != label {
t.Fatalf("label pattern differs: policy %q schema %v compiled %q", label, defs["label"], ctLabelRE.String())
}
if alphabet := ctPolStr("identity.label_alphabet"); len(alphabet) != 32 {
t.Errorf("alphabet %q", alphabet)
}
// the detector a node builds for its own addresses accepts exactly the labels of the vectors
detector := ctHostPattern("devices.all-e2e.test")
for _, v := range ctList(t, ctVectors(t, "labels.json"), "labels") {
got := detector.MatchString(v["value"].(string) + ".devices.all-e2e.test")
if got != (v["ok"] == true) {
t.Errorf("%v: detector says %v", v["id"], got)
}
}
}
func TestContractsSchemaConstantsEqualThePolicy(t *testing.T) {
defs := ctSchemaSet.docs["common.schema.json"].(map[string]any)["$defs"].(map[string]any)
pattern := func(name string) any { return defs[name].(map[string]any)["pattern"] }
for name, key := range map[string]string{"label": "identity.label_regex", "fqdn": "identity.fqdn_regex", "device_id": "identity.device_id_regex",
"stream_id": "identity.stream_id_regex", "attach_secret": "identity.attach_secret_regex", "nonce": "identity.nonce_regex", "jti": "identity.jti_regex",
"kid": "identity.kid_regex"} {
if pattern(name) != ctPolStr(key) {
t.Errorf("schema %s pattern differs from policy %s", name, key)
}
}
for name, key := range map[string]string{"error_code": "protocol.error_codes", "close_reason": "protocol.close_reasons", "source_class": "protocol.source_classes",
"authorize_status": "enums.authorize_status", "budget_class": "enums.budget_class"} {
if !reflect.DeepEqual(ctStrings(defs[name].(map[string]any)["enum"]), ctPolStrings(key)) {
t.Errorf("schema %s enum differs from policy %s", name, key)
}
}
grammar := ctGrammar["types"].(map[string]any)
for _, kind := range []string{"stream_id", "attach_secret", "nonce"} {
if grammar[kind].(map[string]any)["pattern"] != pattern(kind) {
t.Errorf("grammar type %s differs from the schema", kind)
}
}
}
func TestContractsLabelVectors(t *testing.T) {
doc := ctVectors(t, "labels.json")
for _, v := range ctList(t, doc, "labels") {
if got := ctLabelOK(v["value"].(string)); got != (v["ok"] == true) {
t.Errorf("%v: label %q gave %v", v["id"], v["value"], got)
}
}
for _, v := range ctList(t, doc, "fqdns") {
var zones []string
if z, ok := v["zones"]; ok {
zones = ctStrings(z)
}
if got := ctFQDNOK(v["value"].(string), zones); got != (v["ok"] == true) {
t.Errorf("%v: host %q gave %v", v["id"], v["value"], got)
}
}
}
func TestContractsAddressVectors(t *testing.T) {
for _, v := range ctList(t, ctVectors(t, "lines.json"), "ips") {
if got := ctValidIP(v["value"].(string)); got != (v["ok"] == true) {
t.Errorf("%v: %q gave %v", v["id"], v["value"], got)
}
}
}
func TestContractsLineVectors(t *testing.T) {
for _, v := range ctList(t, ctVectors(t, "lines.json"), "lines") {
line, direction := v["line"].(string), v["dir"].(string)
parsed, err := ctParseLine(line, direction)
if v["ok"] == true {
if err != nil {
t.Errorf("%v: %v", v["id"], err)
continue
}
if !ctEqual(parsed, v["parsed"]) {
t.Errorf("%v: parsed %v, want %v", v["id"], parsed, v["parsed"])
}
if v["canonical"] == true {
if again, err := ctFormatLine(parsed); err != nil || again != line {
t.Errorf("%v: canonical form %q (%v), want %q", v["id"], again, err, line)
}
}
continue
}
if err == nil {
t.Errorf("%v: accepted %q", v["id"], truncate(line))
continue
}
if code := err.(*ctProtoErr).code; code != v["error"] {
t.Errorf("%v: error %q, want %q", v["id"], code, v["error"])
}
for _, suffix := range []string{"\n", "\x00", "\r"} {
if _, err := ctParseLine(line+suffix, direction); err == nil {
t.Errorf("%v: accepted with suffix %q", v["id"], suffix)
}
}
}
}
func truncate(s string) string {
if len(s) > 80 {
return s[:80] + "..."
}
return s
}
func TestContractsBodyVectors(t *testing.T) {
doc := ctVectors(t, "bodies.json")
for _, v := range ctList(t, doc, "bodies") {
errs := ctSchemaSet.validate(v["schema"].(string), v["body"])
if (len(errs) == 0) != (v["ok"] == true) {
t.Errorf("%v: errors %v", v["id"], errs)
}
}
for _, v := range ctList(t, doc, "feeds") {
errs := ctCheckFeed(v["body"], v["since"].(float64))
if (len(errs) == 0) != (v["ok"] == true) {
t.Errorf("%v: errors %v", v["id"], errs)
}
}
}
func TestContractsSessionVectors(t *testing.T) {
for _, v := range ctList(t, ctVectors(t, "tokens.json"), "sessions") {
ctx := v["ctx"].(map[string]any)
keys, code := ctParseJWKS(ctx["jwks"])
if code != "" {
t.Fatalf("%v: %s", v["id"], code)
}
token, _ := v["token"].(string) // a null token is "not a string": malformed
var got ctResult
if _, isString := v["token"].(string); !isString {
got = ctFail("malformed_token")
} else {
got = ctVerifySession(token, ctx["now"].(float64), ctx["audience"].(string), keys, ctx["skew"].(float64))
}
ctCheckVerdict(t, v, ctVerdict(got))
}
}
func TestContractsProofVectors(t *testing.T) {
for _, v := range ctList(t, ctVectors(t, "tokens.json"), "proofs") {
ctx := v["ctx"].(map[string]any)
keys, _ := ctParseJWKS(ctx["jwks"])
now, skew := ctx["now"].(float64), ctx["skew"].(float64)
session := ctVerifySession(ctx["session"].(string), now, ctPolStr("identity.aud_tunnel"), keys, skew)
if !session.ok {
t.Fatalf("%v: session %s", v["id"], session.err)
}
seen := map[string]bool{}
for _, id := range ctStrings(ctx["seen"]) {
seen[id] = true
}
got := ctVerifyProof(v["proof"].(string), now, ctx["session"].(string), session.claims, ctx["nonce"].(string), ctx["htu"].(string), skew, seen)
ctCheckVerdict(t, v, ctVerdict(got))
}
}
func TestContractsNodeAssertionVectors(t *testing.T) {
for _, v := range ctList(t, ctVectors(t, "tokens.json"), "assertions") {
ctx := v["ctx"].(map[string]any)
keys, code := ctParseNodeKeys(ctx["node_keys"])
if code != "" {
t.Fatalf("%v: %s", v["id"], code)
}
seen := map[string]bool{}
for _, id := range ctStrings(ctx["seen"]) {
seen[id] = true
}
got := ctVerifyNodeAssertion(v["assertion"].(string), ctx["now"].(float64), keys, ctx["method"].(string), ctx["url"].(string), []byte(ctx["body"].(string)),
ctx["skew"].(float64), seen)
ctCheckVerdict(t, v, ctVerdict(got))
}
}
func TestContractsStatementVectors(t *testing.T) {
for _, v := range ctList(t, ctVectors(t, "tokens.json"), "statements") {
ctx := v["ctx"].(map[string]any)
keys, _ := ctParseJWKS(ctx["jwks"])
got := ctVerifyStatement(v["statement"].(string), ctx["now"].(float64), keys, ctx["device_id"].(string), ctx["last_seq"].(float64))
ctCheckVerdict(t, v, ctVerdict(got))
}
}
func TestContractsSurfaceVectors(t *testing.T) {
for _, v := range ctList(t, ctVectors(t, "tokens.json"), "surface") {
switch v["kind"] {
case "challenge":
ctCheckVerdict(t, v, ctVerdict(ctVerifySurfaceChallenge(v["secret"].(string), v["nonce"].(string), v["mac"].(string))))
case "proof":
ctCheckVerdict(t, v, ctVerdict(ctVerifySurfaceProof(v["secret"].(string), v["nonce"].(string), int(v["pid"].(float64)), int(v["port"].(float64)), v["proof"].(string))))
default:
if got := ctIdentityRequest(v["host"].(string), int(v["port"].(float64)), v["nonce"].(string), v["mac"].(string)); got != v["text"] {
t.Errorf("%v: request %q, want %q", v["id"], got, v["text"])
}
}
}
}
func TestContractsKeySetVectors(t *testing.T) {
for _, v := range ctList(t, ctVectors(t, "tokens.json"), "key_sets") {
var code string
if v["kind"] == "jwks" {
_, code = ctParseJWKS(v["doc"])
} else {
_, code = ctParseNodeKeys(v["doc"])
}
got := map[string]any{"ok": true}
if code != "" {
got = map[string]any{"ok": false, "error": code}
}
ctCheckVerdict(t, v, got)
}
}
func TestContractsTranscriptVectors(t *testing.T) {
for _, v := range ctList(t, ctVectors(t, "transcripts.json"), "transcripts") {
violation := ctCheckTranscript(v)
expect := v["expect"].(map[string]any)
if (violation == "") != (expect["ok"] == true) || (violation != "" && violation != expect["violation"]) {
t.Errorf("%v: violation %q, want %v", v["id"], violation, expect)
}
}
}
// TestContractsGoNodeDoesWhatTheGoldenTranscriptsSay runs the scripted inputs of every golden transcript through an independent Go node
// (the twin of contracts/fake_node.py) and demands the very outputs Python produced: two implementations, one behaviour.
func TestContractsGoNodeDoesWhatTheGoldenTranscriptsSay(t *testing.T) {
replayed := 0
for _, v := range ctList(t, ctVectors(t, "transcripts.json"), "transcripts") {
if v["replay"] != true {
continue
}
replayed++
inputs, golden := ctSplitTranscript(v["events"].([]any))
node := ctNewNodeSim(v["ctx"].(map[string]any))
var produced []map[string]any
for _, input := range inputs {
produced = append(produced, node.handle(input)...)
}
if len(produced) != len(golden) {
t.Errorf("%v: %d outputs, want %d", v["id"], len(produced), len(golden))
continue
}
for i := range golden {
if !ctEqual(toAny(produced[i]), toAny(golden[i])) {
t.Errorf("%v: output %d is %v, want %v", v["id"], i, produced[i], golden[i])
break
}
}
}
if replayed < 8 {
t.Errorf("only %d replayable transcripts", replayed)
}
}
func toAny(m map[string]any) any { return m }
func TestContractsEveryTruncationOfAGoodTokenIsRefused(t *testing.T) {
tokens := ctVectors(t, "tokens.json")
var vector map[string]any
for _, v := range ctList(t, tokens, "proofs") {
if v["id"] == "proof.ok.baseline" {
vector = v
}
}
ctx := vector["ctx"].(map[string]any)
keys, _ := ctParseJWKS(ctx["jwks"])
session := ctx["session"].(string)
claims := ctVerifySession(session, ctx["now"].(float64), "all-tunnel", keys, 5).claims
for _, token := range []string{session, vector["proof"].(string)} {
for cut := 0; cut < len(token)-1; cut += 7 {
if ctVerifySession(token[:cut], ctx["now"].(float64), "all-tunnel", keys, 5).ok {
t.Fatalf("truncated session accepted at %d", cut)
}
if ctVerifyProof(token[:cut], ctx["now"].(float64), session, claims, ctx["nonce"].(string), ctx["htu"].(string), 5, map[string]bool{}).ok {
t.Fatalf("truncated proof accepted at %d", cut)
}
}
for i := 0; i < len(token); i += 11 {
flipped := token[:i] + map[bool]string{true: "B", false: "A"}[token[i] == 'A'] + token[i+1:]
if strings.Count(flipped, ".") == 2 && ctVerifySession(flipped, ctx["now"].(float64), "all-tunnel", keys, 5).ok && flipped != token {
t.Fatalf("corrupted session accepted at %d", i)
}
}
}
}
files/tunnel/contracts_verify_test.go 1643 lines · 47.5 KB
package main
// The Go reference verifier of the ALL 2.0 contracts (the contract): the same checks as contracts/e2e_contracts.py and
// contracts/ts/verify.ts, written independently, so that the three give the same verdict AND the same error word on every golden vector
// (contracts_test.go runs them). It lives in a test file on purpose: WP01 is spec and fixtures only. The node (WP05, WP06) is written in its
// own files and is tested against these functions and against the vectors; every constant is read from contracts/policy.json, never typed.
import (
"crypto/ed25519"
"crypto/hmac"
"crypto/sha256"
"crypto/subtle"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"math"
"net/netip"
"os"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"unicode/utf8"
)
const ctContracts = "../contracts"
// ---------------------------------------------------------------------------------------------------------------------------- JSON
// ctParseJSON parses one JSON value: duplicate object keys are an error (a body with a repeated key means different things to different
// implementations), numbers are float64 and a non-integral one is the caller's business, anything after the value is an error.
func ctParseJSON(text string) (any, error) {
if !utf8.ValidString(text) {
return nil, errors.New("utf-8")
}
dec := json.NewDecoder(strings.NewReader(text))
dec.UseNumber()
value, err := ctDecodeValue(dec)
if err != nil {
return nil, err
}
if _, err := dec.Token(); err != io.EOF {
return nil, errors.New("trailing data")
}
return value, nil
}
func ctDecodeValue(dec *json.Decoder) (any, error) {
token, err := dec.Token()
if err != nil {
return nil, err
}
switch t := token.(type) {
case json.Delim:
if t == '{' {
out := map[string]any{}
for dec.More() {
keyToken, err := dec.Token()
if err != nil {
return nil, err
}
key, ok := keyToken.(string)
if !ok {
return nil, errors.New("key")
}
if _, dup := out[key]; dup {
return nil, errors.New("duplicate key")
}
value, err := ctDecodeValue(dec)
if err != nil {
return nil, err
}
out[key] = value
}
if _, err := dec.Token(); err != nil {
return nil, err
}
return out, nil
}
if t == '[' {
out := []any{}
for dec.More() {
value, err := ctDecodeValue(dec)
if err != nil {
return nil, err
}
out = append(out, value)
}
if _, err := dec.Token(); err != nil {
return nil, err
}
return out, nil
}
return nil, errors.New("delimiter")
case json.Number:
f, err := strconv.ParseFloat(string(t), 64)
if err != nil || math.IsInf(f, 0) {
return nil, errors.New("number")
}
return f, nil
default:
return token, nil // string, bool, nil
}
}
// ctNoFloats refuses a non-integral number anywhere in a value (bodies hold integers, never fractions).
func ctNoFloats(value any) error {
switch v := value.(type) {
case float64:
if v != math.Trunc(v) {
return errors.New("float")
}
case map[string]any:
for _, item := range v {
if err := ctNoFloats(item); err != nil {
return err
}
}
case []any:
for _, item := range v {
if err := ctNoFloats(item); err != nil {
return err
}
}
}
return nil
}
func ctEscape(s string) string {
var b strings.Builder
b.WriteByte('"')
for _, r := range s {
switch {
case r == '"':
b.WriteString(`\"`)
case r == '\\':
b.WriteString(`\\`)
case r == '\n':
b.WriteString(`\n`)
case r == '\r':
b.WriteString(`\r`)
case r == '\t':
b.WriteString(`\t`)
case r == '\b':
b.WriteString(`\b`)
case r == '\f':
b.WriteString(`\f`)
case r < 0x20 || r > 0x7e:
if r > 0xffff {
r -= 0x10000
fmt.Fprintf(&b, `\u%04x\u%04x`, 0xd800+(r>>10), 0xdc00+(r&0x3ff))
} else {
fmt.Fprintf(&b, `\u%04x`, r)
}
default:
b.WriteRune(r)
}
}
b.WriteByte('"')
return b.String()
}
// ctCanonicalJSON is the form of a JSON body on a wire line: keys sorted, no whitespace, ASCII only.
func ctCanonicalJSON(value any) string {
switch v := value.(type) {
case nil:
return "null"
case bool:
return strconv.FormatBool(v)
case float64:
return strconv.FormatInt(int64(v), 10)
case string:
return ctEscape(v)
case []any:
parts := make([]string, len(v))
for i, item := range v {
parts[i] = ctCanonicalJSON(item)
}
return "[" + strings.Join(parts, ",") + "]"
case map[string]any:
keys := make([]string, 0, len(v))
for key := range v {
keys = append(keys, key)
}
sort.Strings(keys)
parts := make([]string, len(keys))
for i, key := range keys {
parts[i] = ctEscape(key) + ":" + ctCanonicalJSON(v[key])
}
return "{" + strings.Join(parts, ",") + "}"
}
panic(fmt.Sprintf("canonical json: %T", value))
}
func ctEqual(a, b any) bool {
switch x := a.(type) {
case nil:
return b == nil
case bool:
y, ok := b.(bool)
return ok && x == y
case float64:
y, ok := b.(float64)
return ok && x == y
case string:
y, ok := b.(string)
return ok && x == y
case []any:
y, ok := b.([]any)
if !ok || len(x) != len(y) {
return false
}
for i := range x {
if !ctEqual(x[i], y[i]) {
return false
}
}
return true
case map[string]any:
y, ok := b.(map[string]any)
if !ok || len(x) != len(y) {
return false
}
for key, item := range x {
other, present := y[key]
if !present || !ctEqual(item, other) {
return false
}
}
return true
}
return false
}
func ctLoadFile(path string) (any, error) {
raw, err := os.ReadFile(path)
if err != nil {
return nil, err
}
return ctParseJSON(string(raw))
}
// ---------------------------------------------------------------------------------------------------------------------------- policy
var ctPolicy = func() map[string]any {
value, err := ctLoadFile(filepath.Join(ctContracts, "policy.json"))
if err != nil {
panic(err)
}
return value.(map[string]any)
}()
// ctPol reads one policy value by its dotted key.
func ctPol(dotted string) any {
var value any = ctPolicy
for _, part := range strings.Split(dotted, ".") {
value = value.(map[string]any)[part]
}
return value
}
func ctPolInt(dotted string) int { return int(ctPol(dotted).(float64)) }
func ctPolStr(dotted string) string { return ctPol(dotted).(string) }
func ctPolStrings(dotted string) []string {
var out []string
for _, item := range ctPol(dotted).([]any) {
out = append(out, item.(string))
}
return out
}
// ctAnchored compiles one of the contracts' ^...$ patterns (RE2: `$` is the end of the text, as in JavaScript).
func ctAnchored(pattern string) *regexp.Regexp {
if !strings.HasPrefix(pattern, "^") || !strings.HasSuffix(pattern, "$") {
panic("contract patterns are anchored: " + pattern)
}
return regexp.MustCompile(pattern)
}
var (
ctLabelRE = ctAnchored(ctPolStr("identity.label_regex"))
ctFQDNRE = ctAnchored(ctPolStr("identity.fqdn_regex"))
ctNonceRE = ctAnchored(ctPolStr("identity.nonce_regex"))
ctKidRE = ctAnchored(ctPolStr("identity.kid_regex"))
ctDeviceIDRE = ctAnchored(ctPolStr("identity.device_id_regex"))
)
func ctLabelOK(value string) bool { return ctLabelRE.MatchString(value) }
func ctFQDNOK(value string, zones []string) bool {
if len(value) > ctPolInt("identity.fqdn_max_length") || !ctFQDNRE.MatchString(value) {
return false
}
if zones == nil {
return true
}
zone := value[strings.Index(value, ".")+1:]
for _, z := range zones {
if z == zone {
return true
}
}
return false
}
// ctHostPattern is the detector a node or a device builds for its own address in a zone.
func ctHostPattern(zone string) *regexp.Regexp {
label := ctPolStr("identity.label_regex")
return regexp.MustCompile("^" + label[1:len(label)-1] + `\.` + regexp.QuoteMeta(zone) + "$")
}
func ctValidIP(value string) bool {
if value == "" || value != strings.ToLower(value) || strings.ContainsAny(value, "%[] \t\r\n/") {
return false
}
_, err := netip.ParseAddr(value)
return err == nil
}
// ---------------------------------------------------------------------------------------------------------------------------- JSON Schema subset
type ctSchemas struct {
docs map[string]any
patterns map[string]*regexp.Regexp
}
func ctLoadSchemas() *ctSchemas {
matches, err := filepath.Glob(filepath.Join(ctContracts, "schemas", "*.schema.json"))
if err != nil || len(matches) == 0 {
panic("no schemas")
}
s := &ctSchemas{docs: map[string]any{}, patterns: map[string]*regexp.Regexp{}}
for _, path := range matches {
doc, err := ctLoadFile(path)
if err != nil {
panic(err)
}
s.docs[filepath.Base(path)] = doc
}
return s
}
func (s *ctSchemas) validate(name string, instance any) []string {
var errs []string
s.check(name, s.docs[name], instance, "", &errs)
return errs
}
func (s *ctSchemas) resolve(doc, ref string) (string, any) {
file, pointer, _ := strings.Cut(ref, "#")
target := file
if target == "" {
target = doc
}
node := s.docs[target]
if pointer != "" {
for _, part := range strings.Split(strings.TrimPrefix(pointer, "/"), "/") {
part = strings.ReplaceAll(strings.ReplaceAll(part, "~1", "/"), "~0", "~")
node = node.(map[string]any)[part]
}
}
return target, node
}
func (s *ctSchemas) pattern(text string) *regexp.Regexp {
if p, ok := s.patterns[text]; ok {
return p
}
p := ctAnchored(text)
s.patterns[text] = p
return p
}
func ctTypeOK(kind string, value any) bool {
switch kind {
case "string":
_, ok := value.(string)
return ok
case "boolean":
_, ok := value.(bool)
return ok
case "null":
return value == nil
case "object":
_, ok := value.(map[string]any)
return ok
case "array":
_, ok := value.([]any)
return ok
case "integer":
f, ok := value.(float64)
return ok && f == math.Trunc(f)
case "number":
_, ok := value.(float64)
return ok
}
return false
}
func (s *ctSchemas) check(doc string, schemaAny any, value any, path string, errs *[]string) {
schema := schemaAny.(map[string]any)
at := path
if at == "" {
at = "/"
}
if ref, ok := schema["$ref"].(string); ok {
target, node := s.resolve(doc, ref)
s.check(target, node, value, path, errs)
}
if want, ok := schema["const"]; ok && !ctEqual(value, want) {
*errs = append(*errs, at+": const")
}
if list, ok := schema["enum"].([]any); ok {
found := false
for _, item := range list {
found = found || ctEqual(value, item)
}
if !found {
*errs = append(*errs, at+": enum")
}
}
if kinds, ok := schema["type"]; ok {
var names []any
if list, isList := kinds.([]any); isList {
names = list
} else {
names = []any{kinds}
}
match := false
for _, name := range names {
match = match || ctTypeOK(name.(string), value)
}
if !match {
*errs = append(*errs, at+": type")
return
}
}
if options, ok := schema["oneOf"].([]any); ok {
hits := 0
for _, option := range options {
var trial []string
s.check(doc, option, value, path, &trial)
if len(trial) == 0 {
hits++
}
}
if hits != 1 {
*errs = append(*errs, at+": oneOf")
}
}
switch v := value.(type) {
case string:
if n, ok := schema["minLength"].(float64); ok && float64(utf8.RuneCountInString(v)) < n {
*errs = append(*errs, at+": minLength")
}
if n, ok := schema["maxLength"].(float64); ok && float64(utf8.RuneCountInString(v)) > n {
*errs = append(*errs, at+": maxLength")
}
if p, ok := schema["pattern"].(string); ok && !s.pattern(p).MatchString(v) {
*errs = append(*errs, at+": pattern")
}
if schema["format"] == "ip" && !ctValidIP(v) {
*errs = append(*errs, at+": format")
}
case float64:
if n, ok := schema["minimum"].(float64); ok && v < n {
*errs = append(*errs, at+": minimum")
}
if n, ok := schema["maximum"].(float64); ok && v > n {
*errs = append(*errs, at+": maximum")
}
case map[string]any:
if required, ok := schema["required"].([]any); ok {
for _, key := range required {
if _, present := v[key.(string)]; !present {
*errs = append(*errs, path+"/"+key.(string)+": required")
}
}
}
props, _ := schema["properties"].(map[string]any)
keys := make([]string, 0, len(v))
for key := range v {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
if sub, ok := props[key]; ok {
s.check(doc, sub, v[key], path+"/"+key, errs)
} else if schema["additionalProperties"] == false {
*errs = append(*errs, path+"/"+key+": additionalProperties")
}
}
case []any:
if n, ok := schema["minItems"].(float64); ok && float64(len(v)) < n {
*errs = append(*errs, at+": minItems")
}
if n, ok := schema["maxItems"].(float64); ok && float64(len(v)) > n {
*errs = append(*errs, at+": maxItems")
}
if schema["uniqueItems"] == true {
for i := range v {
for j := i + 1; j < len(v); j++ {
if ctEqual(v[i], v[j]) {
*errs = append(*errs, at+": uniqueItems")
i, j = len(v), len(v)
}
}
}
}
if items, ok := schema["items"]; ok {
for i, item := range v {
s.check(doc, items, item, path+"/"+strconv.Itoa(i), errs)
}
}
}
}
var ctSchemaSet = ctLoadSchemas()
// ---------------------------------------------------------------------------------------------------------------------------- the wire grammar
type ctProtoErr struct{ code string }
func (e *ctProtoErr) Error() string { return e.code }
var ctGrammar = func() map[string]any {
value, err := ctLoadFile(filepath.Join(ctContracts, "schemas", "wire-v2.grammar.json"))
if err != nil {
panic(err)
}
return value.(map[string]any)
}()
func ctFieldOK(kind, token string) bool {
spec := ctGrammar["types"].(map[string]any)[kind].(map[string]any)
if key, ok := spec["max_bytes_from"].(string); ok && len(token) > int(ctPol(key).(float64)) {
return false
}
if p, ok := spec["pattern"].(string); ok && !ctAnchored(p).MatchString(token) {
return false
}
if spec["format"] == "ip" && !ctValidIP(token) {
return false
}
if key, ok := spec["enum_from"].(string); ok {
found := false
for _, item := range ctPolStrings(key) {
found = found || item == token
}
if !found {
return false
}
}
return true
}
func ctRules(spec map[string]any, key string) []map[string]any {
var out []map[string]any
if list, ok := spec[key].([]any); ok {
for _, item := range list {
out = append(out, item.(map[string]any))
}
}
return out
}
// ctParseLine parses one protocol line (the text before the LF) travelling in `direction` ("d2n" or "n2d"): {"verb": ..., <field>: ...}; HINT
// gives "addresses" and OK gives "body". The error is a *ctProtoErr whose code is the word the node answers with.
func ctParseLine(line, direction string) (map[string]any, error) {
for _, r := range line {
if r < 0x20 || r > 0x7e {
return nil, &ctProtoErr{"bad_command"}
}
}
verb, rest, hasSpace := strings.Cut(line, " ")
var spec map[string]any
if s, ok := ctGrammar["verbs"].(map[string]any)[verb]; ok {
spec = s.(map[string]any)
}
limit := ctPolInt("protocol.line_max_bytes")
if spec != nil {
if key, ok := spec["max"].(string); ok {
limit = int(ctPol(key).(float64))
}
}
if len(line)+1 > limit {
return nil, &ctProtoErr{"line_too_long"}
}
if spec == nil || (spec["dir"] != direction && spec["dir"] != "both") || line == "" || (hasSpace && rest == "") {
return nil, &ctProtoErr{"bad_command"}
}
out := map[string]any{"verb": verb}
if schema, ok := spec["json"].(string); ok {
if !hasSpace {
return nil, &ctProtoErr{"bad_command"}
}
body, err := ctParseJSON(rest)
if err == nil {
err = ctNoFloats(body)
}
if err != nil || len(ctSchemaSet.validate(schema, body)) > 0 {
return nil, &ctProtoErr{"bad_command"}
}
out["body"] = body
return out, nil
}
var tokens []string
if hasSpace {
tokens = strings.Split(rest, " ")
}
for _, token := range tokens {
if token == "" {
return nil, &ctProtoErr{"bad_command"}
}
}
if repeat, ok := spec["repeat"].(map[string]any); ok {
min, max := int(repeat["min"].(float64)), int(ctPol(repeat["max_from"].(string)).(float64))
if len(tokens) < min || len(tokens) > max {
return nil, &ctProtoErr{"bad_command"}
}
addresses := make([]any, len(tokens))
for i, token := range tokens {
if !ctFieldOK(repeat["type"].(string), token) {
return nil, &ctProtoErr{"bad_command"}
}
addresses[i] = token
}
out["addresses"] = addresses
return out, nil
}
fields, optional := ctRules(spec, "fields"), ctRules(spec, "optional")
if len(tokens) < len(fields) || len(tokens) > len(fields)+len(optional) {
return nil, &ctProtoErr{"bad_command"}
}
for i, token := range tokens {
rule := append(append([]map[string]any{}, fields...), optional...)[i]
kind := rule["type"].(string)
if want, ok := rule["equals"].(string); ok && token != want {
if ctFieldOK(kind, token) {
return nil, &ctProtoErr{rule["error_if_number"].(string)}
}
return nil, &ctProtoErr{"bad_command"}
}
if !ctFieldOK(kind, token) {
code := "bad_command"
if custom, ok := rule["error"].(string); ok {
code = custom
}
return nil, &ctProtoErr{code}
}
out[rule["name"].(string)] = token
}
return out, nil
}
// ctFormatLine is the canonical line of a parsed message (the inverse of ctParseLine, which it is checked against).
func ctFormatLine(message map[string]any) (string, error) {
verb := message["verb"].(string)
spec := ctGrammar["verbs"].(map[string]any)[verb].(map[string]any)
var line string
switch {
case spec["json"] != nil:
line = verb + " " + ctCanonicalJSON(message["body"])
case spec["repeat"] != nil:
parts := []string{verb}
for _, address := range message["addresses"].([]any) {
parts = append(parts, address.(string))
}
line = strings.Join(parts, " ")
default:
parts := []string{verb}
for _, rule := range append(ctRules(spec, "fields"), ctRules(spec, "optional")...) {
if value, ok := message[rule["name"].(string)]; ok {
parts = append(parts, value.(string))
}
}
line = strings.Join(parts, " ")
}
direction := spec["dir"].(string)
if direction == "both" {
direction = "d2n"
}
if _, err := ctParseLine(line, direction); err != nil {
return "", err
}
return line, nil
}
// ---------------------------------------------------------------------------------------------------------------------------- tokens
type ctResult struct {
ok bool
err string
claims map[string]any
jti string
}
func ctFail(code string) ctResult { return ctResult{err: code} }
var ctB64Charset = regexp.MustCompile(`^[A-Za-z0-9_-]+$`)
func ctB64Decode(text string) ([]byte, error) {
if !ctB64Charset.MatchString(text) {
return nil, errors.New("base64url")
}
raw, err := base64.RawURLEncoding.Strict().DecodeString(text)
if err != nil || base64.RawURLEncoding.EncodeToString(raw) != text {
return nil, errors.New("base64url")
}
return raw, nil
}
func ctB64Encode(raw []byte) string { return base64.RawURLEncoding.EncodeToString(raw) }
func ctSHA256B64(data []byte) string {
sum := sha256.Sum256(data)
return ctB64Encode(sum[:])
}
type ctJWS struct {
header, claims map[string]any
signature []byte
signingInput []byte
}
// ctJWSParts splits and decodes a compact JWS; any structural fault is "malformed_token". An empty signature part is allowed here (alg none
// then fails as bad_alg, which is the more useful word).
func ctJWSParts(token string, maxBytes int) (*ctJWS, string) {
for _, r := range token {
if r > 0x7f {
return nil, "malformed_token"
}
}
if len(token) > maxBytes {
return nil, "malformed_token"
}
parts := strings.Split(token, ".")
if len(parts) != 3 {
return nil, "malformed_token"
}
decode := func(text string) (map[string]any, bool) {
raw, err := ctB64Decode(text)
if err != nil {
return nil, false
}
value, err := ctParseJSON(string(raw))
if err != nil {
return nil, false
}
object, ok := value.(map[string]any)
return object, ok
}
header, ok1 := decode(parts[0])
claims, ok2 := decode(parts[1])
if !ok1 || !ok2 {
return nil, "malformed_token"
}
var signature []byte
if parts[2] != "" {
raw, err := ctB64Decode(parts[2])
if err != nil {
return nil, "malformed_token"
}
signature = raw
}
for _, name := range []string{"crit", "b64", "jku", "x5u", "x5c"} {
if _, present := header[name]; present {
return nil, "malformed_token"
}
}
if ctNoFloats(claims) != nil {
return nil, "malformed_token"
}
return &ctJWS{header, claims, signature, []byte(parts[0] + "." + parts[1])}, ""
}
func ctEd25519OK(x string, signature, input []byte) bool {
if len(signature) != ed25519.SignatureSize {
return false
}
raw, err := ctB64Decode(x)
if err != nil || len(raw) != ed25519.PublicKeySize {
return false
}
return ed25519.Verify(ed25519.PublicKey(raw), input, signature)
}
func ctStr(m map[string]any, key string) (string, bool) {
s, ok := m[key].(string)
return s, ok
}
func ctNum(m map[string]any, key string) (float64, bool) {
f, ok := m[key].(float64)
return f, ok
}
// ctClaimsOK validates claims by a schema, ignoring the errors of the named keys (checked earlier under their own error word).
func ctClaimsOK(claims map[string]any, schema string, ignore ...string) bool {
for _, e := range ctSchemaSet.validate(schema, claims) {
skipped := false
for _, name := range ignore {
skipped = skipped || strings.HasPrefix(e, "/"+name+":")
}
if !skipped {
return false
}
}
return true
}
func ctJWKX(value any) (string, bool) {
jwk, ok := value.(map[string]any)
if !ok {
return "", false
}
x, hasX := jwk["x"].(string)
if _, private := jwk["d"]; private || jwk["kty"] != "OKP" || jwk["crv"] != "Ed25519" || !hasX || len(x) != 43 {
return "", false
}
raw, err := ctB64Decode(x)
if err != nil || len(raw) != 32 {
return "", false
}
return x, true
}
// ctParseJWKS: kid -> x, or "bad_jwks".
func ctParseJWKS(doc any) (map[string]string, string) {
if len(ctSchemaSet.validate("jwks.schema.json", doc)) > 0 {
return nil, "bad_jwks"
}
keys := map[string]string{}
for _, item := range doc.(map[string]any)["keys"].([]any) {
key := item.(map[string]any)
kid := key["kid"].(string)
if _, dup := keys[kid]; dup {
return nil, "bad_jwks"
}
keys[kid] = key["x"].(string)
}
return keys, ""
}
type ctNodeKey struct{ node, x string }
// ctParseNodeKeys: kid -> (node, x), or "bad_node_keys".
func ctParseNodeKeys(doc any) (map[string]ctNodeKey, string) {
if len(ctSchemaSet.validate("node-keys.schema.json", doc)) > 0 {
return nil, "bad_node_keys"
}
keys := map[string]ctNodeKey{}
for _, item := range doc.(map[string]any)["keys"].([]any) {
key := item.(map[string]any)
kid := key["kid"].(string)
if _, dup := keys[kid]; dup {
return nil, "bad_node_keys"
}
keys[kid] = ctNodeKey{key["node"].(string), key["x"].(string)}
}
return keys, ""
}
// ctSignedByJWKS: the common first half of the session and statement checks (parse, alg, kid, signature, typ).
func ctSignedByJWKS(token string, keys map[string]string, typ string) (*ctJWS, string) {
jws, code := ctJWSParts(token, ctPolInt("protocol.statement_line_max_bytes"))
if code != "" {
return nil, code
}
if jws.header["alg"] != "EdDSA" {
return nil, "bad_alg"
}
kid, isString := ctStr(jws.header, "kid")
if !isString || !ctKidRE.MatchString(kid) {
return nil, "unknown_kid"
}
x, known := keys[kid]
if !known {
return nil, "unknown_kid"
}
if !ctEd25519OK(x, jws.signature, jws.signingInput) {
return nil, "bad_signature"
}
if jws.header["typ"] != typ || jws.claims["typ"] != typ {
return nil, "wrong_type"
}
return jws, ""
}
// ctVerifySession is verify_session of e2e_contracts.py.
func ctVerifySession(token string, now float64, audience string, keys map[string]string, skew float64) ctResult {
if len(token) > ctPolInt("protocol.jwt_max_bytes") {
return ctFail("malformed_token")
}
jws, code := ctSignedByJWKS(token, keys, "device-session")
if code != "" {
return ctFail(code)
}
claims := jws.claims
if v, ok := ctNum(claims, "ver"); !ok || v != 2 {
return ctFail("wrong_type")
}
if claims["iss"] != ctPolStr("identity.issuer") {
return ctFail("wrong_issuer")
}
if claims["aud"] != audience {
return ctFail("wrong_audience")
}
if !ctClaimsOK(claims, "session-claims.schema.json", "aud") {
return ctFail("bad_claims")
}
iat, _ := ctNum(claims, "iat")
exp, _ := ctNum(claims, "exp")
if exp-iat != float64(ctPolInt("authorization.session_ttl_s")) {
return ctFail("bad_lifetime")
}
if iat > now+skew {
return ctFail("not_yet_valid")
}
if exp <= now {
return ctFail("expired")
}
return ctResult{ok: true, claims: claims, jti: claims["jti"].(string)}
}
// ctVerifyProof is verify_proof.
func ctVerifyProof(proof string, now float64, sessionToken string, sessionClaims map[string]any, nonce, htu string, skew float64, seen map[string]bool) ctResult {
if len(proof) > ctPolInt("protocol.jwt_max_bytes") {
return ctFail("malformed_token")
}
jws, code := ctJWSParts(proof, ctPolInt("protocol.jwt_max_bytes"))
if code != "" {
return ctFail(code)
}
if jws.header["alg"] != "EdDSA" {
return ctFail("bad_alg")
}
if jws.header["typ"] != "dpop+jwt" {
return ctFail("wrong_type")
}
x, ok := ctJWKX(jws.header["jwk"])
if !ok {
return ctFail("malformed_token")
}
bound := sessionClaims["cnf"].(map[string]any)["jwk"].(map[string]any)["x"].(string)
if x != bound {
return ctFail("wrong_key")
}
if !ctEd25519OK(x, jws.signature, jws.signingInput) {
return ctFail("bad_signature")
}
claims := jws.claims
switch {
case claims["sub"] != sessionClaims["sub"]:
return ctFail("wrong_subject")
case claims["aud"] != ctPolStr("identity.aud_tunnel"):
return ctFail("wrong_audience")
case claims["htm"] != ctPolStr("protocol.proof_htm"):
return ctFail("wrong_htm")
case claims["htu"] != htu:
return ctFail("wrong_htu")
case claims["ath"] != ctSHA256B64([]byte(sessionToken)):
return ctFail("wrong_ath")
case claims["nonce"] != nonce:
return ctFail("wrong_nonce")
case !ctClaimsOK(claims, "proof-claims.schema.json", "htu"):
return ctFail("bad_claims")
}
iat, _ := ctNum(claims, "iat")
if iat > now+skew {
return ctFail("not_yet_valid")
}
if iat+float64(ctPolInt("authorization.proof_ttl_s")) <= now {
return ctFail("stale_proof")
}
id := claims["jti"].(string)
if seen[id] {
return ctFail("replayed_jti")
}
return ctResult{ok: true, claims: claims, jti: id}
}
// ctVerifyNodeAssertion is verify_node_assertion.
func ctVerifyNodeAssertion(token string, now float64, keys map[string]ctNodeKey, method, url string, body []byte, skew float64, seen map[string]bool) ctResult {
jws, code := ctJWSParts(token, ctPolInt("protocol.jwt_max_bytes"))
if code != "" {
return ctFail(code)
}
if jws.header["alg"] != "EdDSA" {
return ctFail("bad_alg")
}
kid, isString := ctStr(jws.header, "kid")
if !isString || !ctKidRE.MatchString(kid) {
return ctFail("unknown_kid")
}
key, known := keys[kid]
if !known {
return ctFail("unknown_kid")
}
if !ctEd25519OK(key.x, jws.signature, jws.signingInput) {
return ctFail("bad_signature")
}
claims := jws.claims
if v, ok := ctNum(claims, "ver"); jws.header["typ"] != "node-assertion" || claims["typ"] != "node-assertion" || !ok || v != 1 {
return ctFail("wrong_type")
}
switch {
case claims["aud"] != ctPolStr("identity.aud_node_assertion"):
return ctFail("wrong_audience")
case claims["sub"] != key.node:
return ctFail("wrong_subject")
case claims["htm"] != method:
return ctFail("wrong_htm")
case claims["htu"] != url:
return ctFail("wrong_htu")
case !ctClaimsOK(claims, "node-assertion-claims.schema.json", "htm", "htu", "aud", "sub"):
return ctFail("bad_claims")
}
iat, _ := ctNum(claims, "iat")
exp, _ := ctNum(claims, "exp")
switch {
case exp-iat != float64(ctPolInt("authorization.node_assertion_ttl_s")):
return ctFail("bad_lifetime")
case iat > now+skew:
return ctFail("not_yet_valid")
case exp <= now:
return ctFail("expired")
case claims["bh"] != ctSHA256B64(body):
return ctFail("wrong_body_hash")
}
id := claims["jti"].(string)
if seen[id] {
return ctFail("replayed_jti")
}
return ctResult{ok: true, claims: claims, jti: id}
}
// ctVerifyStatement is verify_statement.
func ctVerifyStatement(token string, now float64, keys map[string]string, deviceID string, lastSeq float64) ctResult {
jws, code := ctSignedByJWKS(token, keys, "device-access")
if code != "" {
return ctFail(code)
}
claims := jws.claims
if v, ok := ctNum(claims, "ver"); !ok || v != 1 {
return ctFail("wrong_type")
}
switch {
case claims["iss"] != ctPolStr("identity.issuer"):
return ctFail("wrong_issuer")
case claims["aud"] != ctPolStr("identity.aud_access"):
return ctFail("wrong_audience")
case claims["sub"] != deviceID:
return ctFail("wrong_subject")
}
_, hasPaywall := claims["paywall"]
_, hasPaused := claims["paused"]
_, hasResumed := claims["resumed"]
if !ctClaimsOK(claims, "statement-claims.schema.json", "aud", "sub") || hasPaywall != (claims["mode"] == "paywall") || (hasPaused && hasResumed) {
return ctFail("bad_claims")
}
iat, _ := ctNum(claims, "iat")
exp, _ := ctNum(claims, "exp")
ttl := exp - iat
capName := "authorization.access_ttl_paid_s"
if claims["mode"] == "paywall" {
capName = "authorization.access_ttl_paywall_s"
}
if ttl <= 0 || ttl > float64(ctPolInt(capName)) {
return ctFail("bad_lifetime")
}
if iat > now+float64(ctPolInt("authorization.access_future_slack_s")) {
return ctFail("not_yet_valid")
}
if seq, _ := ctNum(claims, "seq"); seq < lastSeq {
return ctFail("stale_seq")
}
return ctResult{ok: true, claims: claims, jti: claims["jti"].(string)}
}
// ---------------------------------------------------------------------------------------------------------------------------- the feed
// ctCheckFeed is check_feed: the schema and the rules a schema cannot say.
func ctCheckFeed(body any, since float64) []string {
if errs := ctSchemaSet.validate("feed-response.schema.json", body); len(errs) > 0 {
return errs
}
var errs []string
previous := since
doc := body.(map[string]any)
for i, item := range doc["events"].([]any) {
event := item.(map[string]any)
epoch := event["epoch"].(float64)
if epoch <= previous {
errs = append(errs, fmt.Sprintf("/events/%d: order", i))
}
previous = math.Max(previous, epoch)
kind := event["type"].(string)
global := kind == "kill" || kind == "kill_clear"
if global != (event["device_id"] == nil && event["label"] == nil) {
errs = append(errs, fmt.Sprintf("/events/%d: scope", i))
}
if (event["device_id"] == nil) != (event["label"] == nil) {
errs = append(errs, fmt.Sprintf("/events/%d: scope", i))
}
}
if doc["epoch"].(float64) < previous {
errs = append(errs, "/epoch: behind events")
}
return errs
}
// ---------------------------------------------------------------------------------------------------------------------------- surface identity proof
func ctSurfaceKey(secret string) ([]byte, bool) {
raw, err := ctB64Decode(secret)
return raw, err == nil && len(raw) == 32
}
func ctSurfaceMAC(secret, nonce string) (string, bool) {
key, ok := ctSurfaceKey(secret)
if !ok {
return "", false
}
mac := hmac.New(sha256.New, key)
mac.Write([]byte("all-surface-challenge-v1\n" + nonce))
return ctB64Encode(mac.Sum(nil)), true
}
func ctSurfaceProof(secret, nonce string, pid, port int) (string, bool) {
key, ok := ctSurfaceKey(secret)
if !ok {
return "", false
}
mac := hmac.New(sha256.New, key)
mac.Write([]byte("all-surface-proof-v1\n" + nonce + "\n" + strconv.Itoa(pid) + "\n" + strconv.Itoa(port)))
return ctB64Encode(mac.Sum(nil)), true
}
func ctEqualText(a, b string) bool { return subtle.ConstantTimeCompare([]byte(a), []byte(b)) == 1 }
func ctVerifySurfaceChallenge(secret, nonce, mac string) ctResult {
want, ok := ctSurfaceMAC(secret, nonce)
if !ok || !ctNonceRE.MatchString(nonce) || !ctEqualText(want, mac) {
return ctFail("bad_challenge")
}
return ctResult{ok: true}
}
func ctVerifySurfaceProof(secret, nonce string, pid, port int, proof string) ctResult {
want, ok := ctSurfaceProof(secret, nonce, pid, port)
if !ok || !ctNonceRE.MatchString(nonce) || !ctEqualText(want, proof) {
return ctFail("bad_proof")
}
return ctResult{ok: true}
}
func ctIdentityRequest(host string, port int, nonce, mac string) string {
return fmt.Sprintf("GET %s HTTP/1.1\r\nHost: %s:%d\r\n%s: %s\r\n%s: %s\r\nConnection: keep-alive\r\n\r\n", ctPolStr("surface.identity_path"), host, port,
ctPolStr("surface.identity_challenge_header"), nonce, ctPolStr("surface.identity_mac_header"), mac)
}
// ---------------------------------------------------------------------------------------------------------------------------- node behaviour
type ctConn struct {
phase, session, kind, nonce, proof, host, status, sub string
claims map[string]any
sessionOK, authorized, superseded bool
reply map[string]any
validUntil float64
}
// ctHello is a complete ClientHello waiting for its OPEN.
type ctHello struct{ ip, class string }
type ctStream struct {
secret string
t float64
consumed bool
}
func ctText(m map[string]any, key string) string {
s, _ := m[key].(string)
return s
}
// ctCheckTranscript is check_transcript: does the recorded behaviour of one node obey the contract? "" is ok, otherwise the violation word.
func ctCheckTranscript(doc map[string]any) string {
ctx := doc["ctx"].(map[string]any)
keys, code := ctParseJWKS(ctx["jwks"])
if code != "" {
return code
}
skew := ctx["skew"].(float64)
conns := map[string]*ctConn{}
var order []string
seen := map[string]bool{}
streams := map[string]*ctStream{}
secretsUsed := map[string]bool{}
hellos := map[string][]ctHello{} // per host: the peer_ip and class of each complete ClientHello still waiting for its OPEN
ended := false
var now float64
for _, item := range doc["events"].([]any) {
event := item.(map[string]any)
now = event["t"].(float64)
kind := ctText(event, "ev")
var conn *ctConn
if name, ok := event["conn"].(string); ok && (kind == "recv" || kind == "send" || kind == "cp" || kind == "cp_reply") {
if conns[name] == nil {
conns[name] = &ctConn{phase: "new", kind: "hello", proof: "none"}
order = append(order, name)
}
conn = conns[name]
}
switch kind {
case "client_hello":
if event["complete"] == true {
hellos[ctText(event, "sni")] = append(hellos[ctText(event, "sni")], ctHello{ctText(event, "peer_ip"), ctText(event, "class")})
}
case "recv":
message, err := ctParseLine(ctText(event, "line"), "d2n")
if err != nil {
continue
}
verb := message["verb"].(string)
if (verb == "HELLO" && conn.phase == "new") || (verb == "RENEW" && conn.phase == "ready") {
token := message["session"].(string)
result := ctVerifySession(token, now, ctPolStr("identity.aud_tunnel"), keys, skew)
conn.sessionOK = result.ok && (verb == "HELLO" || result.claims["sub"] == conn.sub)
conn.session = token
conn.claims = nil
if result.ok {
conn.claims = result.claims
}
conn.kind = map[bool]string{true: "hello", false: "renew"}[verb == "HELLO"]
conn.nonce = ""
conn.proof, conn.authorized, conn.reply = "none", false, nil
if verb == "HELLO" {
conn.phase = "hello"
}
} else if verb == "PROOF" && conn.sessionOK && conn.nonce != "" {
result := ctVerifyProof(message["proof"].(string), now, conn.session, conn.claims, conn.nonce, ctx["node_url"].(string), skew, seen)
switch {
case result.ok:
conn.proof = "ok"
case result.err == "replayed_jti":
conn.proof = "replayed"
default:
conn.proof = "invalid"
}
conn.authorized, conn.reply = false, nil
if result.ok {
seen[result.jti] = true
}
}
case "cp":
body := event["body"].(map[string]any)
if len(ctSchemaSet.validate("authorize-request.schema.json", body)) > 0 || conn.claims == nil || body["device_id"] != conn.claims["sub"] ||
body["session_jti"] != conn.claims["jti"] || body["reason"] != conn.kind || body["node"] != ctx["node_id"] {
return "bad_authorize_request"
}
if conn.proof == "ok" {
conn.authorized = true
}
case "cp_reply":
conn.reply = nil
if conn.authorized {
conn.reply = event
}
case "send":
message, err := ctParseLine(ctText(event, "line"), "n2d")
if err != nil {
return "bad_line"
}
switch message["verb"].(string) {
case "CHALLENGE":
if !conn.sessionOK || (conn.phase != "hello" && conn.phase != "ready") {
return "challenge_for_invalid_session"
}
conn.nonce = message["nonce"].(string)
case "OK":
if ctText(event, "line") != "OK "+ctCanonicalJSON(message["body"]) {
return "bad_line"
}
switch conn.proof {
case "none":
return "ok_without_proof"
case "replayed":
return "replayed_proof_accepted"
case "invalid":
return "ok_with_invalid_proof"
}
if !conn.authorized || conn.reply == nil || conn.reply["status"].(float64) != 200 {
if conn.kind == "renew" {
return "renew_without_authorize"
}
return "ok_without_authorize"
}
body, answer := message["body"].(map[string]any), conn.reply["body"].(map[string]any)
if body["host"] != answer["host"] {
return "host_not_from_control_plane"
}
if body["status"] != answer["status"] || body["valid_until"] != answer["valid_until"] || !ctEqual(body["policy"], answer["policy"]) || body["node"] != ctx["node_id"] {
return "ok_mismatch_control_plane"
}
conn.phase, conn.host, conn.status = "ready", body["host"].(string), body["status"].(string)
conn.validUntil, conn.sub = body["valid_until"].(float64), conn.claims["sub"].(string)
conn.proof, conn.authorized, conn.reply = "none", false, nil
for _, name := range order {
if other := conns[name]; other != conn && other.phase == "ready" && other.sub == conn.sub {
other.superseded = true
}
}
case "OPEN":
switch {
case conn.phase != "ready" || conn.status == "":
return "open_before_ok"
case conn.status == "paused":
return "open_when_paused"
case now >= conn.validUntil:
return "open_after_valid_until"
case len(hellos[conn.host]) == 0:
for _, list := range hellos {
if len(list) > 0 {
return "open_wrong_host"
}
}
return "open_without_client_hello"
}
waiting := -1
for k, hello := range hellos[conn.host] {
if hello.ip == message["peer_ip"] && hello.class == message["class"] {
waiting = k
break
}
}
if waiting < 0 {
return "open_wrong_client"
}
id, secret := message["stream_id"].(string), message["secret"].(string)
if streams[id] != nil || secretsUsed[secret] {
return "duplicate_stream"
}
hellos[conn.host] = append(hellos[conn.host][:waiting], hellos[conn.host][waiting+1:]...)
streams[id] = &ctStream{secret: secret, t: now}
secretsUsed[secret] = true
case "CLOSE":
conn.phase = "closed"
}
case "attach":
message, err := ctParseLine(ctText(event, "line"), "d2n")
if err != nil {
continue
}
known := streams[message["stream_id"].(string)]
good := known != nil && ctEqualText(known.secret, message["secret"].(string))
fresh := good && !known.consumed && now-known.t <= float64(ctPolInt("protocol.attach_timeout_s"))
if event["result"] == "ok" {
switch {
case known == nil:
return "attach_unknown_accepted"
case !good:
return "attach_wrong_secret_accepted"
case known.consumed:
return "attach_secret_reused"
case !fresh:
return "attach_after_timeout"
}
known.consumed = true
} else if fresh {
return "attach_refused_good_secret"
}
case "end":
ended = true
for _, name := range order {
other := conns[name]
if other.phase == "ready" && other.superseded {
return "replaced_not_closed"
}
if other.phase == "ready" && now > other.validUntil {
return "no_close_at_valid_until"
}
}
}
}
if !ended {
return "transcript_without_end"
}
return ""
}
// ---------------------------------------------------------------------------------------------------------------------------- an active node (the Go twin of contracts/fake_node.py)
type ctSimConn struct {
phase, session, kind, nonce, host, status, sub string
claims map[string]any
waiting bool
validUntil float64
}
type ctNodeSim struct {
nodeID, nodeURL string
keys map[string]string
skew float64
nonces, streamIDs, secrets []string
conns map[string]*ctSimConn
order []string
seen map[string]bool
streams map[string]*ctStream
}
func ctNewNodeSim(ctx map[string]any) *ctNodeSim {
keys, code := ctParseJWKS(ctx["jwks"])
if code != "" {
panic(code)
}
randoms := ctx["randoms"].(map[string]any)
list := func(name string) []string {
var out []string
for _, item := range randoms[name].([]any) {
out = append(out, item.(string))
}
return out
}
return &ctNodeSim{nodeID: ctx["node_id"].(string), nodeURL: ctx["node_url"].(string), keys: keys, skew: ctx["skew"].(float64), nonces: list("nonces"),
streamIDs: list("stream_ids"), secrets: list("secrets"), conns: map[string]*ctSimConn{}, seen: map[string]bool{}, streams: map[string]*ctStream{}}
}
func ctSend(t float64, conn, line string) map[string]any {
return map[string]any{"t": t, "conn": conn, "ev": "send", "line": line}
}
func (n *ctNodeSim) conn(name string) *ctSimConn {
if n.conns[name] == nil {
n.conns[name] = &ctSimConn{phase: "new", kind: "hello"}
n.order = append(n.order, name)
}
return n.conns[name]
}
func ctErrLine(code string) string {
line, err := ctFormatLine(map[string]any{"verb": "ERR", "code": code})
if err != nil {
panic(err)
}
return line
}
func ctErrFor(code, fallback string) string {
switch code {
case "malformed_token":
return "bad_hello"
case "not_yet_valid", "stale_proof":
return "clock_skew"
case "wrong_audience", "expired":
return "auth"
}
return fallback
}
func (n *ctNodeSim) handle(event map[string]any) []map[string]any {
t := event["t"].(float64)
switch event["ev"] {
case "recv":
return n.recv(t, event["conn"].(string), event["line"].(string))
case "cp_reply":
return n.cpReply(t, event["conn"].(string), event["status"].(float64), event["body"])
case "client_hello":
return n.clientHello(t, event)
case "attach":
return n.attach(t, event)
case "tick":
var out []map[string]any
for _, name := range n.order {
if c := n.conns[name]; c.phase == "ready" && t >= c.validUntil {
c.phase = "closed"
out = append(out, ctSend(t, name, "CLOSE expired"))
}
}
return out
}
return nil
}
func (n *ctNodeSim) recv(t float64, name, line string) []map[string]any {
c := n.conn(name)
message, err := ctParseLine(line, "d2n")
if err != nil {
return []map[string]any{ctSend(t, name, ctErrLine(err.(*ctProtoErr).code))}
}
verb := message["verb"].(string)
switch verb {
case "HELLO", "RENEW":
if (verb == "HELLO" && c.phase != "new") || (verb == "RENEW" && c.phase != "ready") { // RENEW only on a live connection
return []map[string]any{ctSend(t, name, ctErrLine("bad_command"))}
}
token := message["session"].(string)
result := ctVerifySession(token, t, ctPolStr("identity.aud_tunnel"), n.keys, n.skew)
if !result.ok {
return []map[string]any{ctSend(t, name, ctErrLine(ctErrFor(result.err, "auth")))}
}
if verb == "RENEW" && result.claims["sub"] != c.sub {
return []map[string]any{ctSend(t, name, ctErrLine("auth"))}
}
c.session, c.claims, c.kind, c.waiting = token, result.claims, "hello", false
if verb == "RENEW" {
c.kind = "renew"
} else {
c.phase = "hello"
}
c.nonce, n.nonces = n.nonces[0], n.nonces[1:]
return []map[string]any{ctSend(t, name, "CHALLENGE "+c.nonce)}
case "PROOF":
if c.nonce == "" {
return []map[string]any{ctSend(t, name, ctErrLine("bad_command"))}
}
result := ctVerifyProof(message["proof"].(string), t, c.session, c.claims, c.nonce, n.nodeURL, n.skew, n.seen)
c.nonce = ""
if !result.ok {
word := "bad_proof"
if result.err != "replayed_jti" {
word = ctErrFor(result.err, "bad_proof")
}
return []map[string]any{ctSend(t, name, ctErrLine(word))}
}
n.seen[result.jti] = true
c.waiting = true
body := map[string]any{"v": 1.0, "device_id": c.claims["sub"], "session_jti": c.claims["jti"], "reason": c.kind, "node": n.nodeID}
return []map[string]any{{"t": t, "conn": name, "ev": "cp", "call": "authorize", "body": body}}
case "PING":
return []map[string]any{ctSend(t, name, "PONG")}
case "HINT":
return nil
case "CLOSE":
c.phase = "closed"
return nil
}
return []map[string]any{ctSend(t, name, ctErrLine("bad_command"))}
}
func (n *ctNodeSim) cpReply(t float64, name string, status float64, body any) []map[string]any {
c := n.conn(name)
if !c.waiting {
return nil
}
c.waiting = false
if status != 200 {
word := "auth"
if status >= 500 || status == 429 {
word = "control_unavailable"
}
if m, ok := body.(map[string]any); ok {
switch m["error"] {
case "license_inactive", "device_revoked", "paused":
word = m["error"].(string)
}
}
return []map[string]any{ctSend(t, name, ctErrLine(word))}
}
answer, _ := body.(map[string]any)
if len(ctSchemaSet.validate("authorize-response.schema.json", body)) > 0 || answer["device_id"] != c.claims["sub"] {
return []map[string]any{ctSend(t, name, ctErrLine("control_unavailable"))}
}
okBody := map[string]any{"v": 2.0, "node": n.nodeID, "host": answer["host"], "status": answer["status"], "valid_until": answer["valid_until"],
"server_time": answer["server_time"], "policy": answer["policy"]}
line, err := ctFormatLine(map[string]any{"verb": "OK", "body": okBody})
if err != nil {
panic(err)
}
out := []map[string]any{ctSend(t, name, line)}
c.phase, c.host, c.status, c.validUntil, c.sub = "ready", answer["host"].(string), answer["status"].(string), answer["valid_until"].(float64), c.claims["sub"].(string)
for _, other := range n.order {
if o := n.conns[other]; other != name && o.phase == "ready" && o.sub == c.sub {
o.phase = "closed"
out = append(out, ctSend(t, other, "CLOSE replaced"))
}
}
return out
}
func (n *ctNodeSim) clientHello(t float64, event map[string]any) []map[string]any {
sni := event["sni"].(string)
alert := []map[string]any{{"t": t, "ev": "tls_alert", "sni": sni}}
if event["complete"] != true {
return alert
}
for _, name := range n.order {
c := n.conns[name]
if c.phase == "ready" && c.host == sni && c.status != "paused" && t < c.validUntil {
id, secret := n.streamIDs[0], n.secrets[0]
n.streamIDs, n.secrets = n.streamIDs[1:], n.secrets[1:]
n.streams[id] = &ctStream{secret: secret, t: t}
line, err := ctFormatLine(map[string]any{"verb": "OPEN", "stream_id": id, "secret": secret, "peer_ip": event["peer_ip"], "class": event["class"]})
if err != nil {
panic(err)
}
return []map[string]any{ctSend(t, name, line)}
}
}
return alert
}
func (n *ctNodeSim) attach(t float64, event map[string]any) []map[string]any {
out := map[string]any{"t": t, "conn": event["conn"], "ev": "attach", "line": event["line"], "result": "err"}
message, err := ctParseLine(event["line"].(string), "d2n")
if err != nil {
return []map[string]any{out}
}
known := n.streams[message["stream_id"].(string)]
if known != nil && known.secret == message["secret"].(string) && !known.consumed && t-known.t <= float64(ctPolInt("protocol.attach_timeout_s")) {
known.consumed = true
out["result"] = "ok"
}
return []map[string]any{out}
}
// ctSplitTranscript is split_transcript: (inputs, outputs) of a golden transcript.
func ctSplitTranscript(events []any) (inputs, outputs []map[string]any) {
for _, item := range events {
event := item.(map[string]any)
switch event["ev"] {
case "recv", "cp_reply", "client_hello", "tick", "end":
inputs = append(inputs, event)
case "attach":
stripped := map[string]any{}
for key, value := range event {
if key != "result" {
stripped[key] = value
}
}
inputs = append(inputs, stripped)
outputs = append(outputs, event)
case "send", "cp", "tls_alert":
outputs = append(outputs, event)
}
}
return inputs, outputs
}
files/tunnel/controlplane.go 496 lines · 19.2 KB
package main
import (
"bytes"
"context"
"crypto/ed25519"
"crypto/tls"
"crypto/x509"
"encoding/json"
"encoding/pem"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"sync"
"time"
)
// The real control-plane client (the contract, sections 3.3 and 4): what the node says to POST /v1/tunnel/authorize and GET /v1/tunnel/feed.
//
// The node holds exactly ONE secret: its own Ed25519 signing key. Every request carries an assertion (header x-all-node-assertion), a 60 s JWS bound to
// the method, the URL and the SHA-256 of the exact body bytes, used once. The control plane has the public half (TUNNEL_NODE_KEYS; removing the kid
// revokes the node). There is no shared secret, no commerce key and no bearer token in this program, and the request a device could influence is only
// the device id and the session id inside the authorize body: never a host name.
const (
authorizePath = "/v1/tunnel/authorize"
feedPath = "/v1/tunnel/feed"
jwksPath = "/.well-known/device-jwks.json"
maxControlBody = 256 << 10 // an answer carries at most a 64 KiB statement; anything bigger is not the control plane
maxJWKSBody = 64 << 10
)
var reErrorWord = regexp.MustCompile(`^[a-z_]{1,48}$`)
// HTTPControlConfig is what the client needs. Everything but the key has a safe default.
type HTTPControlConfig struct {
BaseURL string // https://<control plane>, nothing after the host
NodeID string
KID string // the kid of this node's public key in TUNNEL_NODE_KEYS
Key ed25519.PrivateKey
Client *http.Client // nil: a client of its own (TLS, no proxy, no redirect, bounded connections)
Now func() time.Time // nil: time.Now
Version string // for the User-Agent
}
// HTTPControl is the node's ControlPlane and FeedSource.
type HTTPControl struct {
base *url.URL
hostHTU string // the host as the control plane's runtime writes it in a URL: the port only when it is not 443
nodeID string
kid string
key ed25519.PrivateKey
client *http.Client
now func() time.Time
agent string
}
// NewHTTPControl validates the configuration. It does not touch the network.
func NewHTTPControl(cfg HTTPControlConfig) (*HTTPControl, error) {
u, err := url.Parse(cfg.BaseURL)
if err != nil || u.Scheme != "https" || u.Hostname() == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || (u.Path != "" && u.Path != "/") {
return nil, fmt.Errorf("control plane URL %q: want https://<host>[:port] and nothing else", cfg.BaseURL)
}
if !reNodeID.MatchString(cfg.NodeID) || !reKID.MatchString(cfg.KID) {
return nil, fmt.Errorf("control plane client: bad node id or kid")
}
if len(cfg.Key) != ed25519.PrivateKeySize {
return nil, fmt.Errorf("control plane client: no node signing key")
}
c := &HTTPControl{base: u, nodeID: cfg.NodeID, kid: cfg.KID, key: cfg.Key, client: cfg.Client, now: cfg.Now, agent: "all-tunnel/" + cfg.Version}
c.hostHTU = strings.ToLower(u.Host)
if u.Port() == "443" {
c.hostHTU = strings.ToLower(u.Hostname())
}
if c.client == nil {
c.client = newControlClient()
} else {
// A client that was handed in is used as it is, except that it never follows a redirect: an assertion must not travel to a host nobody configured.
own := *c.client
own.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }
c.client = &own
}
if c.now == nil {
c.now = time.Now
}
return c, nil
}
// newControlClient is the client of the node to its control plane: TLS from the system roots, no proxy from the environment, no redirect followed (a
// redirect would carry an assertion to a host nobody configured) and a bounded pool (authorize may run authorize_max_concurrent at once, plus the feed).
func newControlClient() *http.Client {
limit := policyData.Authorization.AuthorizeMaxConcur
return &http.Client{
Transport: &http.Transport{
Proxy: nil,
DialContext: (&net.Dialer{Timeout: 5 * time.Second, KeepAliveConfig: net.KeepAliveConfig{Enable: true}}).DialContext,
TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS12},
TLSHandshakeTimeout: 5 * time.Second,
ForceAttemptHTTP2: true,
MaxIdleConnsPerHost: limit + 2,
MaxConnsPerHost: limit + 2,
IdleConnTimeout: 90 * time.Second,
},
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
}
}
// compactJWS signs header and claims (canonical JSON, base64url) with key.
func compactJWS(key ed25519.PrivateKey, header, claims map[string]any) string {
input := b64Encode([]byte(canonicalJSON(header))) + "." + b64Encode([]byte(canonicalJSON(claims)))
return input + "." + b64Encode(ed25519.Sign(key, []byte(input)))
}
// assertion is the node's proof for ONE request: the method, the exact URL and the hash of the exact body bytes, 60 s, with a jti nobody reuses.
func (c *HTTPControl) assertion(method, requestURL string, body []byte) string {
iat := c.now().Unix()
return compactJWS(c.key,
map[string]any{"alg": "EdDSA", "typ": "node-assertion", "kid": c.kid},
map[string]any{"typ": "node-assertion", "ver": 1, "sub": c.nodeID, "aud": policyData.Identity.AudNodeAssert, "htm": method, "htu": requestURL,
"iat": iat, "exp": iat + int64(policyData.Authorization.NodeAssertionTTLS), "jti": newNonce(), "bh": sha256B64(body)})
}
// do sends one signed request and returns the status and the body (at most maxControlBody). A transport failure is a plain error; any answer, including
// an error answer, is a status.
func (c *HTTPControl) do(ctx context.Context, method, pathAndQuery string, body []byte) (int, []byte, error) {
requestURL := "https://" + c.hostHTU + pathAndQuery
req, err := http.NewRequestWithContext(ctx, method, c.base.Scheme+"://"+c.base.Host+pathAndQuery, bytes.NewReader(body))
if err != nil {
return 0, nil, err
}
req.Header.Set("x-all-node-assertion", c.assertion(method, requestURL, body))
req.Header.Set("accept", "application/json")
req.Header.Set("user-agent", c.agent)
if method == http.MethodPost {
req.Header.Set("content-type", "application/json")
}
return c.send(req, maxControlBody)
}
func (c *HTTPControl) send(req *http.Request, limit int64) (int, []byte, error) {
resp, err := c.client.Do(req)
if err != nil {
return 0, nil, err
}
defer resp.Body.Close()
data, err := io.ReadAll(io.LimitReader(resp.Body, limit+1))
if err != nil {
return 0, nil, err
}
if int64(len(data)) > limit {
return resp.StatusCode, nil, errors.New("control plane answer too large")
}
return resp.StatusCode, data, nil
}
// controlError turns a non-200 answer into a ControlError: the closed word of the error body when it is one (never any other text from the network),
// bad_response when it is not.
func controlError(status int, data []byte) *ControlError {
word := "bad_response"
if value, err := parseJSON(string(data)); err == nil {
if m, ok := value.(map[string]any); ok && m["ok"] == false {
if s, ok := m["error"].(string); ok && reErrorWord.MatchString(s) {
word = s
}
}
}
return &ControlError{Status: status, Code: word}
}
// Authorize asks the control plane what the device the node authenticated may do. The body carries the device id, the session's jti, the reason and
// the node: never a host name. Anything but a well-formed 200 is an error the node maps to a closed word (errForControl).
func (c *HTTPControl) Authorize(ctx context.Context, req AuthorizeRequest) (AuthorizeResponse, error) {
body := []byte(canonicalJSON(req.toMap()))
status, data, err := c.do(ctx, http.MethodPost, authorizePath, body)
if err != nil {
return AuthorizeResponse{}, err
}
if status != http.StatusOK {
return AuthorizeResponse{}, controlError(status, data)
}
resp, err := parseAuthorizeResponse(data)
if err != nil {
return AuthorizeResponse{}, &ControlError{Status: http.StatusBadGateway, Code: "bad_response"}
}
return resp, nil
}
// parseAuthorizeResponse reads a 200 answer of authorize (authorize-response.schema.json). The fields the node uses are checked strictly; a field it
// does not know is ignored (a newer control plane may add one).
func parseAuthorizeResponse(data []byte) (AuthorizeResponse, error) {
value, err := parseJSON(string(data))
if err != nil {
return AuthorizeResponse{}, err
}
m, ok := value.(map[string]any)
if !ok || m["ok"] != true {
return AuthorizeResponse{}, errors.New("authorize answer shape")
}
if v, ok := claimInt(m, "v"); !ok || v != 1 {
return AuthorizeResponse{}, errors.New("authorize answer version")
}
var a AuthorizeResponse
var okDevice, okLabel, okHost, okStatus, okPlan bool
a.DeviceID, okDevice = claimString(m, "device_id")
a.Label, okLabel = claimString(m, "label")
a.Host, okHost = claimString(m, "host")
a.Status, okStatus = claimString(m, "status")
a.Plan, okPlan = claimString(m, "plan")
validUntil, okUntil := epochOK(m["valid_until"])
serverTime, okTime := epochOK(m["server_time"])
policy, okPolicy := checkPolicyBlock(m["policy"])
if !okDevice || !reDeviceID.MatchString(a.DeviceID) || !okLabel || !okHost || !okStatus || !okPlan || !okUntil || !okTime || !okPolicy {
return AuthorizeResponse{}, errors.New("authorize answer fields")
}
a.ValidUntil, a.ServerTime, a.Policy = validUntil, serverTime, policy
if access, present := m["access"]; present {
text, ok := access.(string)
if !ok || statementLine(text) == "" {
return AuthorizeResponse{}, errors.New("authorize answer access")
}
a.Access = text
}
if nodes, present := m["nodes"]; present {
if _, ok := nodes.([]any); !ok {
return AuthorizeResponse{}, errors.New("authorize answer nodes")
}
a.Nodes = nodes
}
if wave, ok := claimInt(m, "e2e_wave"); ok && wave >= 0 && wave <= 100 {
a.E2EWave = int(wave)
}
a.E2EKill, _ = m["e2e_kill"].(bool)
a.CopyE2EE, _ = m["copy_e2ee"].(bool)
return a, nil
}
// ---------------------------------------------------------------------------------------------------------------------------- the feed
// Types of a feed event (policy enums.feed_event_type). kill and kill_clear are global; every other event names a device and its label.
const (
feedRevoke = "revoke"
feedPause = "pause"
feedResume = "resume"
feedLicenseEnd = "license_end"
feedRelease = "release"
feedKill = "kill"
feedKillClear = "kill_clear"
)
// FeedEvent is one entry of GET /v1/tunnel/feed (feed-response.schema.json).
type FeedEvent struct {
Epoch int64
Type string
DeviceID string // empty for a global event
Label string // empty for a global event
At int64 // epoch seconds
}
func (e FeedEvent) global() bool { return e.Type == feedKill || e.Type == feedKillClear }
// FeedAnswer is the answer of the feed: Epoch is where the node's cursor goes next. Reset says the cursor is not one the control plane can serve from
// (older than its history, or ahead of it). Skipped counts the events of the answer the node could not read (an unknown type or a malformed field).
type FeedAnswer struct {
Epoch int64
ServerTime int64
Reset bool
Events []FeedEvent
Skipped int
}
// FeedSource is the node's side of GET /v1/tunnel/feed. since is the node's cursor; wait asks the control plane to hold an empty answer until an event
// arrives (at most policy timing.feed_poll_s).
type FeedSource interface {
Feed(ctx context.Context, since int64, wait time.Duration) (FeedAnswer, error)
}
// Feed polls the feed. The URL is the one that is signed, exactly: https://<host><path>?since=<n>[&wait=<s>].
func (c *HTTPControl) Feed(ctx context.Context, since int64, wait time.Duration) (FeedAnswer, error) {
query := "?since=" + strconv.FormatInt(since, 10)
if w := int64(wait / time.Second); w > 0 {
query += "&wait=" + strconv.FormatInt(min(w, int64(policyData.Timing.FeedPollS)), 10)
}
status, data, err := c.do(ctx, http.MethodGet, feedPath+query, nil)
if err != nil {
return FeedAnswer{}, err
}
if status != http.StatusOK {
return FeedAnswer{}, controlError(status, data)
}
answer, err := parseFeedAnswer(data)
if err != nil {
return FeedAnswer{}, &ControlError{Status: http.StatusBadGateway, Code: "bad_response"}
}
return answer, nil
}
// parseFeedAnswer reads a feed answer. The envelope is read strictly (a node must not act on an answer it cannot make sense of); an event it cannot
// read, or whose type it does not know, is skipped and counted, because the rest of the answer is still true and a revoke must not wait for it.
func parseFeedAnswer(data []byte) (FeedAnswer, error) {
value, err := parseJSON(string(data))
if err != nil {
return FeedAnswer{}, err
}
m, ok := value.(map[string]any)
if !ok || m["ok"] != true {
return FeedAnswer{}, errors.New("feed answer shape")
}
if v, ok := claimInt(m, "v"); !ok || v != 1 {
return FeedAnswer{}, errors.New("feed answer version")
}
epoch, okEpoch := claimInt(m, "epoch")
serverTime, okTime := epochOK(m["server_time"])
list, okList := m["events"].([]any)
if !okEpoch || epoch < 0 || epoch > 1<<53-1 || !okTime || !okList || len(list) > feedMaxEvents {
return FeedAnswer{}, errors.New("feed answer fields")
}
answer := FeedAnswer{Epoch: epoch, ServerTime: serverTime}
if reset, present := m["reset"]; present {
if answer.Reset, ok = reset.(bool); !ok {
return FeedAnswer{}, errors.New("feed answer reset")
}
}
for _, item := range list {
event, ok := parseFeedEvent(item)
if !ok {
answer.Skipped++
continue
}
answer.Events = append(answer.Events, event)
}
return answer, nil
}
const feedMaxEvents = 1000 // feed-response.schema.json: events maxItems
func parseFeedEvent(item any) (FeedEvent, bool) {
m, ok := item.(map[string]any)
if !ok {
return FeedEvent{}, false
}
epoch, okEpoch := claimInt(m, "epoch")
at, okAt := epochOK(m["at"])
kind, okKind := claimString(m, "type")
if !okEpoch || epoch < 1 || epoch > 1<<53-1 || !okAt || !okKind || !inList(policyData.Enums.FeedEventType, kind) {
return FeedEvent{}, false
}
event := FeedEvent{Epoch: epoch, Type: kind, At: at}
device, label := m["device_id"], m["label"]
if event.global() {
return event, device == nil && label == nil
}
event.DeviceID, _ = device.(string)
event.Label, _ = label.(string)
return event, reDeviceID.MatchString(event.DeviceID) && reLabel.MatchString(event.Label)
}
// FetchJWKS reads the control plane's published keys. The document is public, so the request carries no assertion; it is read from the configured
// control plane only, never from a URL an answer names, and a redirect is not followed.
func (c *HTTPControl) FetchJWKS(ctx context.Context) ([]byte, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.base.Scheme+"://"+c.base.Host+jwksPath, nil)
if err != nil {
return nil, err
}
req.Header.Set("accept", "application/json")
req.Header.Set("user-agent", c.agent)
status, data, err := c.send(req, maxJWKSBody)
if err != nil {
return nil, err
}
if status != http.StatusOK {
return nil, &ControlError{Status: status, Code: "jwks_unavailable"}
}
return data, nil
}
// ---------------------------------------------------------------------------------------------------------------------------- the node's key
// LoadNodeKey reads the node's Ed25519 signing key: a PKCS#8 PEM file that only its owner may read (a key a group or the world can read is refused,
// the way ssh refuses one). The one exception is the copy systemd hands the service (LoadCredential=): systemd 255 makes it 0440 root:root with an
// ACL for the unit's dynamic user, inside $CREDENTIALS_DIRECTORY, which only that unit can open; there the group bits are the ACL mask, not a group,
// so only world bits are refused.
func LoadNodeKey(path string) (ed25519.PrivateKey, error) {
st, err := os.Stat(path)
if err != nil {
return nil, fmt.Errorf("node key: %w", err)
}
refused := os.FileMode(0o077)
if dir := os.Getenv("CREDENTIALS_DIRECTORY"); dir != "" && filepath.IsAbs(dir) && filepath.Dir(filepath.Clean(path)) == filepath.Clean(dir) {
refused = 0o007
}
if !st.Mode().IsRegular() || st.Mode().Perm()&refused != 0 {
return nil, fmt.Errorf("node key %s must be a regular file readable by its owner only (mode 0600)", path)
}
raw, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("node key: %w", err)
}
block, _ := pem.Decode(raw)
if block == nil || block.Type != "PRIVATE KEY" {
return nil, fmt.Errorf("node key %s is not a PKCS#8 PEM private key", path)
}
parsed, err := x509.ParsePKCS8PrivateKey(block.Bytes)
if err != nil {
return nil, fmt.Errorf("node key %s: %w", path, err)
}
key, ok := parsed.(ed25519.PrivateKey)
if !ok {
return nil, fmt.Errorf("node key %s is not an Ed25519 key", path)
}
return key, nil
}
// GenerateNodeKey creates the node's key at path (mode 0600, never overwriting a file that is there) and returns it.
func GenerateNodeKey(path string) (ed25519.PrivateKey, error) {
_, key, err := ed25519.GenerateKey(nil)
if err != nil {
return nil, err
}
der, err := x509.MarshalPKCS8PrivateKey(key)
if err != nil {
return nil, err
}
f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
if err != nil {
return nil, fmt.Errorf("node key: %w", err)
}
if _, err := f.Write(pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der})); err != nil {
_ = f.Close()
_ = os.Remove(path)
return nil, err
}
return key, f.Close()
}
// NodeKeysEntry is the TUNNEL_NODE_KEYS document (node-keys.schema.json) that registers this node's public key with the control plane. Printing it is
// all the node does to "publish" its key: the deployment owner sets the variable (the design plan, section 9); revoking the node is removing
// the kid from it.
func NodeKeysEntry(nodeID, kid string, key ed25519.PrivateKey) string {
pub := key.Public().(ed25519.PublicKey)
entry, _ := json.Marshal(map[string]any{"keys": []any{map[string]any{"kid": kid, "node": nodeID, "kty": "OKP", "crv": "Ed25519", "x": b64Encode(pub)}}})
return string(entry)
}
// ---------------------------------------------------------------------------------------------------------------------------- reachability
// cpHealth is what the node has seen of its control plane's reachability: the moment of the last answer of any kind (an error answer such as a 402 is
// an answer: the control plane is up) and of the last failure to get one (a transport error, a timeout, a 5xx, a 429, an answer it could not read).
// The paid grace asks it, so that one slow answer never decides alone.
type cpHealth struct {
mu sync.Mutex
lastGood time.Time
lastBad time.Time
}
func (h *cpHealth) good(now time.Time) {
h.mu.Lock()
h.lastGood = now
h.mu.Unlock()
}
func (h *cpHealth) bad(now time.Time) {
h.mu.Lock()
h.lastBad = now
h.mu.Unlock()
}
// down reports whether the last thing the node saw was a failure and nothing has answered for at least quiet.
func (h *cpHealth) down(now time.Time, quiet time.Duration) bool {
h.mu.Lock()
defer h.mu.Unlock()
return h.lastBad.After(h.lastGood) && now.Sub(h.lastGood) >= quiet
}
// record files the outcome of one call to the control plane.
func (h *cpHealth) record(now time.Time, err error) {
if err == nil || isRefusal(err) {
h.good(now)
return
}
h.bad(now)
}
// isRefusal reports whether err is the control plane saying no (a 4xx other than 429): it answered, so it is reachable.
func isRefusal(err error) bool {
var ce *ControlError
return errors.As(err, &ce) && ce.Status >= 400 && ce.Status < 500 && ce.Status != http.StatusTooManyRequests
}
files/tunnel/controlplane_test.go 514 lines · 19.6 KB
package main
import (
"bytes"
"context"
"crypto/ed25519"
"crypto/rand"
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
)
// The real control-plane client against a fake control plane over HTTPS that authenticates the node with the contracts' own verifier.
func TestControlPlane_AuthorizeCarriesOneAssertionBoundToTheRequestAndNothingElse(t *testing.T) {
e, cp := httpNode(t, nil, nil)
d := e.newDevice()
hc := cp.control()
resp, err := hc.Authorize(context.Background(), AuthorizeRequest{DeviceID: d.id, SessionJTI: randomHex(12), Reason: "hello", Node: testNodeID})
if err != nil {
t.Fatal(err)
}
if resp.DeviceID != d.id || resp.Host != d.host || resp.Status != statusActive || resp.Plan != "paid" || resp.ValidUntil <= e.now().Unix() {
t.Fatalf("answer %+v", resp)
}
cp.mu.Lock()
defer cp.mu.Unlock()
if len(cp.bodies) != 1 || len(cp.headers) < 1 || cp.refused != 0 {
t.Fatalf("authenticated %d requests (%d authorize), refused %d", len(cp.headers), len(cp.bodies), cp.refused)
}
// The assertion is the node's ONLY credential, on authorize and on the feed alike: no bearer, no cookie, no shared secret, no commerce key.
for _, h := range cp.headers {
if h.Get("x-all-node-assertion") == "" {
t.Error("a request without its assertion was authenticated")
}
for name := range h {
switch strings.ToLower(name) {
case "x-all-node-assertion", "content-type", "accept", "user-agent", "content-length", "accept-encoding":
default:
t.Errorf("a request of the node carries the header %s", name)
}
}
}
}
func TestControlPlane_AnAssertionServesOneRequestOnly(t *testing.T) {
_, cp := httpNode(t, nil, nil)
hc := cp.control()
body := []byte(`{"v":1}`)
token := hc.assertion("POST", "https://"+cp.srv.Listener.Addr().String()+authorizePath, body)
cp.mu.Lock()
keys := cp.nodeKeys
cp.mu.Unlock()
url := "https://" + cp.srv.Listener.Addr().String() + authorizePath
seen := map[string]bool{}
first := ctVerifyNodeAssertion(token, epochSeconds(time.Now()), keys, "POST", url, body, 5, seen)
if !first.ok {
t.Fatalf("assertion refused: %s", first.err)
}
seen[first.jti] = true
if r := ctVerifyNodeAssertion(token, epochSeconds(time.Now()), keys, "POST", url, body, 5, seen); r.err != "replayed_jti" {
t.Fatalf("a replayed assertion: %s", r.err)
}
// bound to the method, the URL and the exact body bytes
if r := ctVerifyNodeAssertion(token, epochSeconds(time.Now()), keys, "GET", url, body, 5, map[string]bool{}); r.err != "wrong_htm" {
t.Fatalf("another method: %s", r.err)
}
if r := ctVerifyNodeAssertion(token, epochSeconds(time.Now()), keys, "POST", url+"?x=1", body, 5, map[string]bool{}); r.err != "wrong_htu" {
t.Fatalf("another URL: %s", r.err)
}
if r := ctVerifyNodeAssertion(token, epochSeconds(time.Now()), keys, "POST", url, []byte(`{"v":2}`), 5, map[string]bool{}); r.err != "wrong_body_hash" {
t.Fatalf("another body: %s", r.err)
}
// 60 s, no more
if r := ctVerifyNodeAssertion(token, epochSeconds(time.Now().Add(61*time.Second)), keys, "POST", url, body, 5, map[string]bool{}); r.err != "expired" {
t.Fatalf("an old assertion: %s", r.err)
}
// every call mints a new one (a retry is a new request)
if again := hc.assertion("POST", url, body); again == token {
t.Fatal("two assertions are the same token")
}
}
func TestControlPlane_ABadNodeSignatureIsRefusedByTheControlPlaneAndTheDeviceIsTurnedAway(t *testing.T) {
e, cp := httpNode(t, nil, nil)
d := e.newDevice()
_, rogueKey, _ := ed25519.GenerateKey(rand.Reader)
for name, hc := range map[string]*HTTPControl{
"a key the control plane does not know": func() *HTTPControl {
c, err := NewHTTPControl(HTTPControlConfig{BaseURL: cp.url(), NodeID: testNodeID, KID: cp.nodeKID, Key: rogueKey, Client: cp.srv.Client()})
if err != nil {
t.Fatal(err)
}
return c
}(),
"a kid it does not know": func() *HTTPControl {
c, err := NewHTTPControl(HTTPControlConfig{BaseURL: cp.url(), NodeID: testNodeID, KID: "node-t1-revoked", Key: cp.nodeKey, Client: cp.srv.Client()})
if err != nil {
t.Fatal(err)
}
return c
}(),
"another node's id": func() *HTTPControl {
c, err := NewHTTPControl(HTTPControlConfig{BaseURL: cp.url(), NodeID: "t2", KID: cp.nodeKID, Key: cp.nodeKey, Client: cp.srv.Client()})
if err != nil {
t.Fatal(err)
}
return c
}(),
"a clock a minute off": func() *HTTPControl {
c, err := NewHTTPControl(HTTPControlConfig{BaseURL: cp.url(), NodeID: testNodeID, KID: cp.nodeKID, Key: cp.nodeKey, Client: cp.srv.Client(),
Now: func() time.Time { return e.now().Add(-2 * time.Minute) }})
if err != nil {
t.Fatal(err)
}
return c
}(),
} {
_, err := hc.Authorize(context.Background(), AuthorizeRequest{DeviceID: d.id, SessionJTI: randomHex(12), Reason: "hello", Node: testNodeID})
ce, ok := err.(*ControlError)
if !ok || ce.Status != 401 || ce.Code != "unauthorized" {
t.Errorf("%s: %v", name, err)
}
if _, err := hc.Feed(context.Background(), 0, 0); err == nil {
t.Errorf("%s: the feed answered", name)
}
}
if e.cp.callsFor(d.id, "") != 0 {
t.Fatal("the control plane decided something for a request it could not authenticate")
}
// A node whose key was revoked (its kid removed from TUNNEL_NODE_KEYS) cannot authorize anyone: the device is turned away with a closed word.
cp.mu.Lock()
delete(cp.nodeKeys, cp.nodeKID)
cp.mu.Unlock()
dc := e.connect(d)
line := dc.hello(func() string { return e.session(d, nil) }, func(s, n string) string { return e.proof(d, s, n, nil) })
if line != "ERR control_unavailable" {
t.Fatalf("a device behind a node the control plane refuses must be told to try later, not that it is wrong: %q", line)
}
dc.expectClosed(2 * time.Second)
if e.srv.Online(d.host) {
t.Fatal("the host is served although nothing authorized it")
}
}
func TestControlPlane_TheNodeHoldsNoSecretButItsOwnKey(t *testing.T) {
// The production binary's whole configuration surface: one key file, no shared secret, no token, no commerce key.
bin := productionBinary(t)
out, _ := exec.Command(bin, "-h").CombinedOutput()
help := strings.ToLower(string(out))
for _, word := range []string{"secret", "token", "hmac", "commerce", "password", "bearer", "api-key", "apikey"} {
if strings.Contains(help, word) {
t.Errorf("the node's flags mention %q:\n%s", word, out)
}
}
if !strings.Contains(help, "-node-key-file") {
t.Fatalf("no node key flag:\n%s", out)
}
for _, name := range []string{"TUNNEL_COMMERCE_KEY", "TUNNEL_SECRET", "TUNNEL_TOKEN", "COMMERCE_HMAC_KEY"} {
if strings.Contains(string(out), name) {
t.Errorf("the node reads %s", name)
}
}
}
func TestControlPlane_ErrorsAreClosedWordsAndNeverTextFromTheNetwork(t *testing.T) {
e, cp := httpNode(t, nil, nil)
d := e.newDevice()
hc := cp.control()
ask := func() error {
_, err := hc.Authorize(context.Background(), AuthorizeRequest{DeviceID: d.id, SessionJTI: randomHex(12), Reason: "renew", Node: testNodeID})
return err
}
code := func(err error) (int, string) {
ce, ok := err.(*ControlError)
if !ok {
return 0, err.Error()
}
return ce.Status, ce.Code
}
e.cp.set(d.id, &devicePlan{host: d.host, err: &ControlError{Status: 402, Code: codeLicenseInactive}})
if st, w := code(ask()); st != 402 || w != codeLicenseInactive {
t.Errorf("402: %d %s", st, w)
}
e.cp.set(d.id, &devicePlan{host: d.host, err: &ControlError{Status: 401, Code: codeDeviceRevoked}})
if st, w := code(ask()); st != 401 || w != codeDeviceRevoked {
t.Errorf("401: %d %s", st, w)
}
cp.setDown(503)
if st, w := code(ask()); st != 503 || w != "end_to_end_unavailable" || errForControl(ask()) != codeControlUnavailable {
t.Errorf("503: %d %s", st, w)
}
cp.setDown(0)
// A control plane that answers with something that is not an error body: bad_response, whatever the text says.
for name, serve := range map[string]http.HandlerFunc{
"html": func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(502)
_, _ = w.Write([]byte("<html>Bad gateway: secret-detail</html>"))
},
"text word": func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(500)
_, _ = w.Write([]byte(`{"ok":false,"error":"Not A Word!"}`))
},
"200 junk": func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte(`{"ok":true}`)) },
"200 other device": func(w http.ResponseWriter, r *http.Request) {
a := AuthorizeResponse{DeviceID: randomHex(16), Label: label20(), Status: statusActive, Plan: "paid"}
a.Host = a.Label + "." + testZone
a.ValidUntil, a.ServerTime = e.now().Unix()+3600, e.now().Unix()
a.Policy = okPolicy{20, 45, 1800, 14400, 300}
_, _ = w.Write([]byte(canonicalJSON(a.toMap())))
},
"too large": func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write(bytes.Repeat([]byte("x"), maxControlBody+10))
},
} {
srv := httptest.NewTLSServer(serve)
other, err := NewHTTPControl(HTTPControlConfig{BaseURL: srv.URL, NodeID: testNodeID, KID: cp.nodeKID, Key: cp.nodeKey, Client: srv.Client()})
if err != nil {
t.Fatal(err)
}
_, aerr := other.Authorize(context.Background(), AuthorizeRequest{DeviceID: d.id, SessionJTI: randomHex(12), Reason: "hello", Node: testNodeID})
srv.Close()
if aerr == nil {
// the answer for another device parses; the node's own check (checkAuthorize) is what refuses it, below
if name != "200 other device" {
t.Errorf("%s: accepted", name)
}
continue
}
if ce, ok := aerr.(*ControlError); ok && strings.Contains(ce.Code, "secret") {
t.Errorf("%s: text from the network became a word: %q", name, ce.Code)
}
if errForControl(aerr) != codeControlUnavailable && name != "text word" {
t.Errorf("%s: %q", name, errForControl(aerr))
}
}
}
func TestControlPlane_ARedirectIsNeverFollowed(t *testing.T) {
e, cp := httpNode(t, nil, nil)
d := e.newDevice()
var landed atomic.Int64
elsewhere := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
landed.Add(1)
_, _ = w.Write([]byte("{}"))
}))
defer elsewhere.Close()
cp.mu.Lock()
cp.redirect = elsewhere.URL + authorizePath
cp.mu.Unlock()
_, err := cp.control().Authorize(context.Background(), AuthorizeRequest{DeviceID: d.id, SessionJTI: randomHex(12), Reason: "hello", Node: testNodeID})
if err == nil || landed.Load() != 0 {
t.Fatalf("a redirect was followed: err=%v landed=%d", err, landed.Load())
}
}
func TestControlPlane_AnAnswerThatIsNotWhatWasAskedIsNeverRelayed(t *testing.T) {
// The control plane answers for another device: the node refuses to relay it (checkAuthorize) and the device is told control_unavailable.
e, _ := httpNode(t, nil, nil)
d := e.newDevice()
e.cp.set(d.id, &devicePlan{host: d.host, answer: func(int) AuthorizeResponse {
a := AuthorizeResponse{DeviceID: randomHex(16), Label: label20(), Status: statusActive, Plan: "paid", Nodes: []any{}}
a.Host = a.Label + "." + testZone
a.ValidUntil, a.ServerTime = e.now().Unix()+3600, e.now().Unix()
a.Policy = okPolicy{20, 45, 1800, 14400, 300}
return a
}})
dc := e.connect(d)
line := dc.hello(func() string { return e.session(d, nil) }, func(s, n string) string { return e.proof(d, s, n, nil) })
if line != "ERR control_unavailable" || e.srv.Online(d.host) {
t.Fatalf("an answer for another device: %q", line)
}
}
func TestControlPlane_TheHostNeverComesFromTheRequest(t *testing.T) {
e, cp := httpNode(t, nil, nil)
d := e.newDevice()
if _, err := cp.control().Authorize(context.Background(), AuthorizeRequest{DeviceID: d.id, SessionJTI: randomHex(12), Reason: "hello", Node: testNodeID}); err != nil {
t.Fatal(err)
}
cp.mu.Lock()
bodies := append([][]byte(nil), cp.bodies...)
cp.mu.Unlock()
if len(bodies) != 1 {
t.Fatalf("bodies %d", len(bodies))
}
doc, perr := parseJSON(string(bodies[0]))
if perr != nil {
t.Fatal(perr)
}
m := doc.(map[string]any)
if len(m) != 5 || m["host"] != nil || m["label"] != nil {
t.Fatalf("the authorize body %s", bodies[0])
}
if errs := ctSchemaSet.validate("authorize-request.schema.json", doc); len(errs) > 0 {
t.Fatalf("not an authorize-request: %v", errs)
}
}
func TestControlPlane_ConfigurationIsValidated(t *testing.T) {
_, priv, _ := ed25519.GenerateKey(rand.Reader)
good := HTTPControlConfig{BaseURL: "https://api.example.test", NodeID: "t1", KID: "k1", Key: priv}
if _, err := NewHTTPControl(good); err != nil {
t.Fatal(err)
}
for name, mutate := range map[string]func(*HTTPControlConfig){
"plain http": func(c *HTTPControlConfig) { c.BaseURL = "http://api.example.test" },
"a path": func(c *HTTPControlConfig) { c.BaseURL = "https://api.example.test/v1" },
"credentials": func(c *HTTPControlConfig) { c.BaseURL = "https://user:pw" + "@api.example.test" },
"a query": func(c *HTTPControlConfig) { c.BaseURL = "https://api.example.test?x=1" },
"no host": func(c *HTTPControlConfig) { c.BaseURL = "https://" },
"bad node id": func(c *HTTPControlConfig) { c.NodeID = "T1" },
"bad kid": func(c *HTTPControlConfig) { c.KID = "" },
"no key": func(c *HTTPControlConfig) { c.Key = nil },
"a short key": func(c *HTTPControlConfig) { c.Key = priv[:10] },
"garbage": func(c *HTTPControlConfig) { c.BaseURL = "::not a url" },
"another scheme": func(c *HTTPControlConfig) { c.BaseURL = "ftp://api.example.test" },
"whitespace host": func(c *HTTPControlConfig) { c.BaseURL = "https://api example.test" },
} {
c := good
mutate(&c)
if _, err := NewHTTPControl(c); err == nil {
t.Errorf("%s: accepted", name)
}
}
// the port 443 is not part of the URL the control plane's runtime sees
c := good
c.BaseURL = "https://api.example.test:443"
hc, err := NewHTTPControl(c)
if err != nil || hc.hostHTU != "api.example.test" {
t.Fatalf("hostHTU %q err %v", hc.hostHTU, err)
}
c.BaseURL = "https://api.example.test:8443"
if hc, _ = NewHTTPControl(c); hc.hostHTU != "api.example.test:8443" {
t.Fatalf("hostHTU %q", hc.hostHTU)
}
}
func TestNodeKey_FileRulesAndTheEntryThatRegistersIt(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "node.key")
key, err := GenerateNodeKey(path)
if err != nil {
t.Fatal(err)
}
st, _ := os.Stat(path)
if st.Mode().Perm() != 0o600 {
t.Fatalf("mode %v", st.Mode())
}
if _, err := GenerateNodeKey(path); err == nil {
t.Fatal("a key was overwritten")
}
loaded, err := LoadNodeKey(path)
if err != nil || !loaded.Equal(key) {
t.Fatalf("load: %v", err)
}
// a key that others can read is refused, as ssh refuses it
if err := os.Chmod(path, 0o644); err != nil {
t.Fatal(err)
}
if _, err := LoadNodeKey(path); err == nil {
t.Fatal("a world-readable key was accepted")
}
// systemd's credential copy is 0440 with an ACL for the unit's user: accepted only inside $CREDENTIALS_DIRECTORY, and never world-readable
if err := os.Chmod(path, 0o440); err != nil {
t.Fatal(err)
}
if _, err := LoadNodeKey(path); err == nil {
t.Fatal("a group-readable key outside the credentials directory was accepted")
}
t.Setenv("CREDENTIALS_DIRECTORY", dir)
if loaded, err := LoadNodeKey(path); err != nil || !loaded.Equal(key) {
t.Fatalf("the systemd credential copy was refused: %v", err)
}
if err := os.Chmod(path, 0o444); err != nil {
t.Fatal(err)
}
if _, err := LoadNodeKey(path); err == nil {
t.Fatal("a world-readable key in the credentials directory was accepted")
}
t.Setenv("CREDENTIALS_DIRECTORY", filepath.Join(dir, "elsewhere"))
if err := os.Chmod(path, 0o440); err != nil {
t.Fatal(err)
}
if _, err := LoadNodeKey(path); err == nil {
t.Fatal("a group-readable key outside the credentials directory was accepted")
}
t.Setenv("CREDENTIALS_DIRECTORY", "")
_ = os.Chmod(path, 0o600)
for name, content := range map[string]string{"garbage": "not pem", "a certificate": "-----BEGIN CERTIFICATE-----\nAAAA\n-----END CERTIFICATE-----\n"} {
bad := filepath.Join(dir, name)
if err := os.WriteFile(bad, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
if _, err := LoadNodeKey(bad); err == nil {
t.Errorf("%s: accepted", name)
}
}
if _, err := LoadNodeKey(filepath.Join(dir, "missing")); err == nil {
t.Fatal("a missing key was accepted")
}
// the entry is a valid TUNNEL_NODE_KEYS document of the contracts, for exactly this key
entry := NodeKeysEntry("t1", "node-t1-1", key)
doc, perr := parseJSON(entry)
if perr != nil {
t.Fatal(perr)
}
keys, word := ctParseNodeKeys(doc)
if word != "" || keys["node-t1-1"].node != "t1" || keys["node-t1-1"].x != b64Encode(key.Public().(ed25519.PublicKey)) {
t.Fatalf("entry %s -> %v %q", entry, keys, word)
}
var round map[string]any
if json.Unmarshal([]byte(entry), &round) != nil {
t.Fatal("entry is not JSON")
}
}
func TestProductionBinary_InitPrintsTheRegistrationEntryAndRefusesWithoutAKey(t *testing.T) {
bin := productionBinary(t)
dir := t.TempDir()
keyPath := filepath.Join(dir, "node.key")
run := func(args ...string) (string, int) {
cmd := exec.Command(bin, args...)
var out, errb bytes.Buffer
cmd.Stdout, cmd.Stderr = &out, &errb
err := cmd.Run()
code := 0
if ee, ok := err.(*exec.ExitError); ok {
code = ee.ExitCode()
}
return out.String() + errb.String(), code
}
out, code := run("-node-id", "t1", "-node-kid", "node-t1-1", "-node-key-file", keyPath, "-node-key-init")
if code != 0 {
t.Fatalf("init: %d %s", code, out)
}
doc, err := parseJSON(strings.TrimSpace(out))
if err != nil {
t.Fatalf("init printed %q", out)
}
if _, word := ctParseNodeKeys(doc); word != "" {
t.Fatalf("init printed an entry the contracts refuse: %s", word)
}
if out2, code := run("-node-id", "t1", "-node-kid", "node-t1-1", "-node-key-file", keyPath, "-node-key-print"); code != 0 || strings.TrimSpace(out2) != strings.TrimSpace(out) {
t.Fatalf("print: %d %q", code, out2)
}
if _, code := run("-node-id", "t1", "-node-kid", "node-t1-1", "-node-key-file", keyPath, "-node-key-init"); code == 0 {
t.Fatal("init overwrote the key")
}
if out, code := run("-node-id", "t1", "-node-host", testNodeHost); code == 0 || !strings.Contains(out, "signing key") {
t.Fatalf("without a key: %d %s", code, out)
}
if out, code := run("-node-id", "t1", "-node-host", testNodeHost, "-node-key-file", keyPath, "-node-kid", "node-t1-1", "-control-url", "http://plain.example.test"); code == 0 || !strings.Contains(out, "https") {
t.Fatalf("plain http control plane: %d %s", code, out)
}
}
// ---------------------------------------------------------------------------------------------------------------------------- authorize concurrency over HTTP
func TestControlPlane_AuthorizeNeverRunsMoreThanFiftyAtOnceOverHTTP(t *testing.T) {
if policyData.Authorization.AuthorizeMaxConcur != 50 {
t.Fatalf("the plan says 50 concurrent authorizations, the policy says %d", policyData.Authorization.AuthorizeMaxConcur)
}
e, cp := httpNode(t, nil, func(c *Config) { c.HelloTimeout, c.FirstLineTimeout = 10*time.Second, 20*time.Second })
cp.mu.Lock()
cp.delay = 80 * time.Millisecond
cp.mu.Unlock()
const devices = 160
var all []*testDevice
for i := 0; i < devices; i++ {
all = append(all, e.newDevice())
}
var connected atomic.Int64
var wg sync.WaitGroup
for _, d := range all {
wg.Add(1)
go func(d *testDevice) {
defer wg.Done()
// (a burst this size can overflow the listener's backlog: a device that is reset tries again, as one does)
for attempt := 0; attempt < 40; attempt++ {
line := helloOnce(e, d)
if strings.HasPrefix(line, "OK ") {
connected.Add(1)
return
}
if line != "ERR control_unavailable" && !strings.HasPrefix(line, "unreachable") {
t.Errorf("a device was refused: %q", line)
return
}
time.Sleep(time.Duration(20+attempt*5) * time.Millisecond)
}
}(d)
}
wg.Wait()
if connected.Load() != devices {
t.Fatalf("%d of %d connected", connected.Load(), devices)
}
if peak := cp.maxInflight(); peak > 50 || peak < 2 {
t.Fatalf("the control plane saw %d authorizations at once; the node's bound is 50", peak)
}
}
files/tunnel/deploy/all-tunnel-cert-check 27 lines · 1.6 KB
#!/bin/sh
# Runs after every `certbot renew` of the node's own certificate (ExecStartPost of all-tunnel-certbot.service) and fails when the certificate is not
# healthy. certbot exits 0 when nothing was due, and also when a renewal "worked" but nothing downstream took it, so a green renewal proves nothing: this
# looks at what the node is actually serving.
# - the certificate file must have more than ALL_TUNNEL_CERT_MIN_DAYS left (the first of policy.monitoring.node_cert_alert_days: 14);
# - when /metrics can be read, the certificate IN SERVICE must too (a renewed file that was never reloaded is the other silent failure).
# Success clears the failure marker all-tunnel-certbot-failed.service writes.
set -eu
here=$(dirname "$0")
. "${ALL_TUNNEL_LIB:-$here/all-tunnel-cert-lib.sh}"
min_days="${ALL_TUNNEL_CERT_MIN_DAYS:-}"
[ -n "$min_days" ] || die "ALL_TUNNEL_CERT_MIN_DAYS is not set"
cert=$(env_value TUNNEL_CERT_FILE)
[ -n "$cert" ] || die "$ALL_TUNNEL_ENV does not say TUNNEL_CERT_FILE"
[ -r "$cert" ] || die "the node certificate $cert cannot be read"
openssl x509 -in "$cert" -noout -checkend $((min_days * 86400)) >/dev/null ||
die "the node certificate has less than $min_days days left: the renewal is not happening (journalctl -u all-tunnel-certbot)"
left=$(metric all_tunnel_node_cert_seconds_left)
if [ -n "$left" ] && [ "$left" -lt $((min_days * 86400)) ]; then
die "the node serves a certificate with less than $min_days days left although the file on disk is newer: reload the node (systemctl reload all-tunnel)"
fi
rm -f "${ALL_TUNNEL_STATE_DIR:-/var/lib/all-tunnel}/certbot.failed"
say info "node certificate has more than $min_days days left"
files/tunnel/deploy/all-tunnel-cert-hook 64 lines · 3.8 KB
#!/bin/sh
# certbot --deploy-hook for the ALL tunnel node's OWN certificate. certbot runs it after every successful renewal with RENEWED_LINEAGE set to the
# directory of the new certificate. It never trusts that the files are good and never reports success it did not see:
# 1. the new pair is checked: it is a certificate for THIS node's name, not about to expire, and its key is the key of the certificate;
# 2. it is installed atomically (key first, then chain; the node reads files it can use and refuses a pair that does not match, so a reader never takes half);
# 3. the node is told to reload (systemctl reload sends SIGHUP: no restart, no dropped handshake, the streams that are open keep their certificate);
# 4. when the node's /metrics can be read, it must now report THIS certificate's end. A node that kept the old one is a failure, not a log line.
# Any failure exits non-zero, so certbot fails, all-tunnel-certbot.service fails and all-tunnel-certbot-failed.service shouts (see that unit).
set -eu
umask 077
here=$(dirname "$0")
. "${ALL_TUNNEL_LIB:-$here/all-tunnel-cert-lib.sh}"
lineage="${RENEWED_LINEAGE:-}"
[ -n "$lineage" ] || die "RENEWED_LINEAGE is not set: this script is certbot's deploy hook"
src_cert="$lineage/fullchain.pem"
src_key="$lineage/privkey.pem"
[ -r "$src_cert" ] && [ -r "$src_key" ] || die "the renewed files are missing in $lineage"
host=$(env_value TUNNEL_NODE_HOST)
dst_cert=$(env_value TUNNEL_CERT_FILE)
dst_key=$(env_value TUNNEL_KEY_FILE)
[ -n "$host" ] && [ -n "$dst_cert" ] && [ -n "$dst_key" ] || die "$ALL_TUNNEL_ENV does not say TUNNEL_NODE_HOST, TUNNEL_CERT_FILE and TUNNEL_KEY_FILE"
openssl x509 -in "$src_cert" -noout -checkend 86400 >/dev/null || die "the renewed certificate is already expired or expires within a day"
openssl x509 -in "$src_cert" -noout -text | grep -Eq "DNS:$host(,|[[:space:]]|\$)" || die "the renewed certificate does not carry this node's name"
cert_key=$(openssl x509 -in "$src_cert" -noout -pubkey | openssl sha256)
file_key=$(openssl pkey -in "$src_key" -pubout | openssl sha256)
[ "$cert_key" = "$file_key" ] || die "the renewed key does not belong to the renewed certificate"
install_atomic() { # install_atomic MODE SOURCE DESTINATION: copy beside the destination, then rename over it
mode=$1; from=$2; to=$3
dir=$(dirname "$to")
[ -d "$dir" ] || install -d -m 0755 "$dir"
tmp="$to.new.$$"
install -m "$mode" "$from" "$tmp"
owner=$(stat -c %u:%g "$dir" 2>/dev/null || true) # the node's dynamic user owns its state directory; the files must be readable by it
if [ -n "$owner" ] && [ "$(id -u)" -eq 0 ]; then chown "$owner" "$tmp"; fi
mv -f "$tmp" "$to"
}
install_atomic 0600 "$src_key" "$dst_key"
install_atomic 0644 "$src_cert" "$dst_cert"
if systemctl is-active --quiet "$ALL_TUNNEL_UNIT"; then
systemctl reload "$ALL_TUNNEL_UNIT" || die "the node did not accept the reload"
expected=$(not_after_epoch "$src_cert") || expected=""
if [ -n "$expected" ]; then
seen=""
tries=${ALL_TUNNEL_RELOAD_TRIES:-10}
i=0
while [ "$i" -lt "$tries" ]; do
i=$((i + 1))
seen=$(metric all_tunnel_node_cert_not_after_seconds)
[ -z "$seen" ] && break # /metrics cannot be read: the monitor and ct_watch.py still watch the certificate the node serves
[ "$seen" = "$expected" ] && break
sleep "${ALL_TUNNEL_RELOAD_PAUSE:-1}"
done
if [ -n "$seen" ] && [ "$seen" != "$expected" ]; then die "the node reloaded but still reports a certificate that ends at $seen, not $expected"; fi
[ -n "$seen" ] || say warning "the node's /metrics cannot be read: the new certificate was installed and the node told to reload, but not confirmed"
fi
else
say warning "the node is not running: the new certificate is installed, and the node reads it when it starts"
fi
say info "node certificate renewed and in service for $host"
files/tunnel/deploy/all-tunnel-cert-lib.sh 34 lines · 2.0 KB
# Shared by all-tunnel-cert-hook and all-tunnel-cert-check (installed next to them in /usr/local/sbin). Sourced, never run.
#
# Everything the node's certificate scripts need comes from the node's own environment file (the one systemd reads), so there is one place that says where the
# certificate is and what the node is called:
# TUNNEL_NODE_HOST the node's own public name TUNNEL_CERT_FILE / TUNNEL_KEY_FILE where the node reads its certificate and key
# TUNNEL_METRICS_ADDR the loopback address of /metrics (default 127.0.0.1:9101)
# Overrides for tests: ALL_TUNNEL_ENV (the file), ALL_TUNNEL_UNIT (the unit to reload), ALL_TUNNEL_LOGGER (the logger command).
ALL_TUNNEL_ENV="${ALL_TUNNEL_ENV:-/etc/all-tunnel/env}"
ALL_TUNNEL_UNIT="${ALL_TUNNEL_UNIT:-all-tunnel}"
env_value() { # env_value KEY [DEFAULT]: the last KEY=value line of the environment file (no quotes, no expansion: it is a systemd EnvironmentFile)
value=$(sed -n "s/^$1=//p" "$ALL_TUNNEL_ENV" 2>/dev/null | tail -n 1)
if [ -n "$value" ]; then printf '%s\n' "$value"; else printf '%s\n' "${2:-}"; fi
}
say() { # say PRIORITY MESSAGE: to the journal when logger exists, always to stderr (what systemd and certbot capture)
printf 'all-tunnel-cert: %s\n' "$2" >&2
if command -v "${ALL_TUNNEL_LOGGER:-logger}" >/dev/null 2>&1; then "${ALL_TUNNEL_LOGGER:-logger}" -p "daemon.$1" -t all-tunnel-cert "$2" || true; fi
}
die() { say crit "$1"; exit 1; }
# not_after_epoch FILE: the end of the certificate in Unix seconds (GNU date on the node, BSD date where a developer runs the tests)
not_after_epoch() {
text=$(openssl x509 -in "$1" -noout -enddate | sed 's/^notAfter=//')
date -u -d "$text" +%s 2>/dev/null || date -u -j -f '%b %e %H:%M:%S %Y %Z' "$text" +%s 2>/dev/null || return 1
}
# metric NAME: one value of the node's loopback /metrics, empty when it cannot be read
metric() {
addr=$(env_value TUNNEL_METRICS_ADDR 127.0.0.1:9101)
curl -fsS --max-time 5 "http://$addr/metrics" 2>/dev/null | sed -n "s/^$1 //p" | head -n 1
}
files/tunnel/deploy/all-tunnel-certbot-failed.service 14 lines · 1.2 KB
# Runs when all-tunnel-certbot.service fails (OnFailure=). It makes the failure impossible to miss on the node itself:
# a CRITICAL line in the journal (`journalctl -p crit -t all-tunnel-certbot`), a marker file that `the deploy tool status` reports, and a wall
# message to whoever is logged in. The service itself stays in the failed state (`systemctl --failed`) until the next run succeeds, which clears the
# marker (all-tunnel-cert-check does).
# The mail to the owner does not come from here (a node has no mail and no secret to send one with): it comes from the site monitor (14 and 5 days left,
# the site's monitor) and from the independent CT watcher, which both read the certificate the node SERVES.
[Unit]
Description=The ALL tunnel node's certificate renewal FAILED
[Service]
Type=oneshot
ExecStart=/usr/bin/logger -p daemon.crit -t all-tunnel-certbot "node certificate renewal FAILED: read journalctl -u all-tunnel-certbot, fix it, then systemctl start all-tunnel-certbot"
ExecStart=/bin/sh -c 'date -u +%%FT%%TZ > /var/lib/all-tunnel/certbot.failed'
ExecStart=-/usr/bin/wall "all-tunnel: the node certificate renewal FAILED (journalctl -u all-tunnel-certbot)"
files/tunnel/deploy/all-tunnel-certbot.service 25 lines · 1.7 KB
# Renews the node's OWN certificate (the name the devices connect to, ALPN all-tunnel/2) with certbot, HTTP-01 through the webroot the node serves on :80.
# Devices' names never come here: their certificates are issued on the devices over DNS-01 (the design plan, section 4).
#
# It fails LOUDLY, on purpose, and in three places a silent failure would hide:
# certbot renew exits non-zero when a renewal was due and failed (no `|| true`, no `-q` that swallows the reason)
# --deploy-hook all-tunnel-cert-hook: checks the new pair, installs it, reloads the node, and fails when the node did not take it
# ExecStartPost all-tunnel-cert-check: fails when the certificate in service has less than ALL_TUNNEL_CERT_MIN_DAYS left, which is what
# a renewal that "succeeded" but never replaced anything looks like
# and any failure runs all-tunnel-certbot-failed.service (a critical journal line, a marker file `tunnel_deploy.py status` reports). The days are the
# first alert of policy.monitoring.node_cert_alert_days (14): the site monitor mails the owner at 14 and at 5 days anyway, this is the node's own voice.
[Unit]
Description=Renew the ALL tunnel node's own certificate (certbot, HTTP-01 webroot)
Wants=network-online.target
After=network-online.target
OnFailure=all-tunnel-certbot-failed.service
[Service]
Type=oneshot
EnvironmentFile=/etc/all-tunnel/env
Environment=ALL_TUNNEL_CERT_MIN_DAYS=14
ExecStart=/usr/bin/certbot renew --non-interactive --no-random-sleep-on-renew --deploy-hook /usr/local/sbin/all-tunnel-cert-hook
ExecStartPost=/usr/local/sbin/all-tunnel-cert-check
TimeoutStartSec=15min
PrivateTmp=yes
NoNewPrivileges=yes
files/tunnel/deploy/all-tunnel-certbot.timer 14 lines · 597 B
# Twice a day, with a random delay so two nodes never renew in the same minute (a renewal reloads the node: see all-tunnel-certbot.service).
# Persistent: a node that was off at the time renews as soon as it is up. certbot itself only renews a certificate inside its renewal window, so most
# runs do nothing; what must never happen is a run that fails and nobody hears of it.
[Unit]
Description=Twice daily check of the ALL tunnel node's own certificate
[Timer]
OnCalendar=*-*-* 03,15:00:00
RandomizedDelaySec=1h
Persistent=true
Unit=all-tunnel-certbot.service
[Install]
WantedBy=timers.target
files/tunnel/deploy/all-tunnel.service 67 lines · 3.7 KB
# systemd unit for the ALL tunnel node (Ubuntu 24.04). `the deploy tool bootstrap` installs it with everything around it (the node runbook):
#
# /opt/all-tunnel/releases/<id>/all-tunnel one directory per release, named by the first 16 hex characters of the binary's SHA-256
# /opt/all-tunnel/current -> releases/<id> what runs; /opt/all-tunnel/previous is what a rollback flips back to
# /usr/local/bin/all-tunnel -> /opt/all-tunnel/current/all-tunnel
# /etc/all-tunnel/env TUNNEL_NODE_ID, TUNNEL_NODE_KID, TUNNEL_NODE_HOST and the certificate paths (0600)
# /etc/all-tunnel/node.key the node's one secret, made ON the node (`all-tunnel -node-key-init`); only its public half is ever printed
#
# Two listeners and no other: :443 (devices' control and data connections on the node's own name, clients of devices on every other name) and :80 (the
# node's own ACME webroot, nothing else). The node holds no secret of devices: it verifies their sessions against the control plane's published keys
# and asks the control plane what each may do, signing every request with its own Ed25519 key, the only secret it has: systemd hands it over as a
# credential (LoadCredential below), so the service's dynamic user reads it and nobody else does. The control plane's address defaults to the one the
# policy pins (TUNNEL_CONTROL_URL changes it for a staging node).
#
# /metrics and /healthz listen on 127.0.0.1:9101 (TUNNEL_METRICS_ADDR changes the port, and refuses an address that is not loopback): aggregate
# counters only, nothing per device, nothing with an address or a name in it. The kernel side of the limits is nftables.conf and sysctl.conf next to
# this file; the node does not load them (it has no privilege to). Nothing the node writes to the journal names an address or a SNI, and
# journald-volatile.conf keeps the journal in memory anyway: nothing of a node survives a reboot but its key and its certificate.
#
# systemctl stop / restart sends SIGTERM, which DRAINS: no new clients, the open streams finish, and the node leaves when they are gone or after the
# quiet period (TUNNEL_DRAIN_QUIET, 20 minutes by default, policy timing.drain_quiet_s), so TimeoutStopSec must be longer than that
# (the deploy tests hold the two together). systemctl reload re-reads the node's certificate (SIGHUP). Deploy one node at a time, after
# taking its address out of the DNS (the design plan, section 5, the node runbook).
[Unit]
Description=ALL tunnel node (SNI router, never decrypts)
After=network-online.target
Wants=network-online.target
[Service]
EnvironmentFile=/etc/all-tunnel/env
Environment=TUNNEL_NODE_KEY_FILE=%d/node-key
Environment=TUNNEL_OPENAI_PREFIXES_CACHE=%S/all-tunnel/openai-prefixes.json
LoadCredential=node-key:/etc/all-tunnel/node.key
ExecStart=/usr/local/bin/all-tunnel
ExecReload=/bin/kill -HUP $MAINPID
Restart=always
RestartSec=2
DynamicUser=yes
StateDirectory=all-tunnel
AmbientCapabilities=CAP_NET_BIND_SERVICE
CapabilityBoundingSet=CAP_NET_BIND_SERVICE
NoNewPrivileges=yes
ProtectSystem=strict
ProtectHome=yes
PrivateTmp=yes
PrivateDevices=yes
ProtectKernelTunables=yes
ProtectKernelModules=yes
ProtectKernelLogs=yes
ProtectControlGroups=yes
ProtectClock=yes
ProtectHostname=yes
ProtectProc=invisible
# no ProcSubset=pid: it hides /proc/sys, and Go reads /proc/sys/net/core/somaxconn for its listen backlog (sysctl.conf raises it to 4096 for the reconnection
# storm); without the file the node falls back to 128
LockPersonality=yes
RestrictNamespaces=yes
RestrictRealtime=yes
RestrictSUIDSGID=yes
RestrictAddressFamilies=AF_INET AF_INET6
SystemCallArchitectures=native
UMask=0077
LimitNOFILE=65536
TimeoutStopSec=1500
[Install]
WantedBy=multi-user.target
files/tunnel/deploy/journald-volatile.conf 12 lines · 625 B
# journald drop-in for an ALL tunnel node: /etc/systemd/journald.conf.d/all-tunnel.conf (the design plan, section 5, "Higiene de logs").
#
# The journal lives in memory only. The node's own log lines carry no address, no name and no label (tunnel/loghygiene_test.go), and this makes sure
# nothing else of the machine (sshd, the kernel, certbot) leaves a persistent trace either: a reboot or a rebuild starts with an empty journal.
# What stays is bounded, so a flood cannot fill the disk or the RAM.
[Journal]
Storage=volatile
RuntimeMaxUse=64M
RuntimeMaxFileSize=8M
MaxRetentionSec=3day
ForwardToSyslog=no
ForwardToWall=no
files/tunnel/deploy/nftables.conf 60 lines · 4.0 KB
#!/usr/sbin/nft -f
# nftables fragment for an ALL tunnel node (the design plan, section 5). It protects the PROCESS, not the product: the node decides who may
# have what (tunnel/limits.go), and these limits sit above its own on purpose, so that the kernel refuses only what the node would never have served:
#
# per source (an IPv4 address, or an IPv6 /64, as the node counts them)
# connections held 1024 the node's cap is node_limits.per_ip_concurrent (16), 8 times that for an address of the OpenAI list or of a device.
# The kernel counts EVERY connection a source holds: a device's control connection plus one data connection per open
# stream (up to node_limits.label_streams_max for each device behind one address), which the node stops counting when
# they attach. 1024 is room for 16 devices at their cap behind one NAT
# new connections 200/s the node's is node_limits.per_ip_new_per_s (10), 8 times that for the OpenAI list or a device
# whole node
# new connections 20000/s and 40000 of burst: a flood the process could not read fast enough anyway
# port 80 (the node's own ACME webroot and nothing else): 32 connections and 20 new a second per source
#
# nftables_test.go (TestNftables_*) checks that these numbers are never below the node's, so a retuned policy cannot leave the kernel stricter than the
# node. The file only ADDS drops: its chain accepts by default, so loading it cannot lock the operator out. SYN cookies are a sysctl, not an nftables
# rule: install deploy/sysctl.conf next to this file.
#
# Check it before loading it: nft -c -f /etc/nftables.d/all-tunnel.conf
# Load it with the system's own nftables service (include it from /etc/nftables.conf), never with a script of the node: the node holds no privilege.
#
# NOTE: written against nftables 1.0 (Ubuntu 24.04). It has been read, not run: the first deployment runs the check above and fixes what it says.
# Loading the file twice must not stack its rules: the table is dropped and made again (the first line makes sure it exists, so the delete never fails).
table inet all_tunnel
delete table inet all_tunnel
table inet all_tunnel {
# connections held per source, counted by the kernel's conntrack
set held4 { type ipv4_addr; flags dynamic; size 65536; }
set held6 { type ipv6_addr; flags dynamic; size 65536; }
# new connections per source and second
set rate4 { type ipv4_addr; flags dynamic,timeout; timeout 1m; size 65536; }
set rate6 { type ipv6_addr; flags dynamic,timeout; timeout 1m; size 65536; }
set held80_4 { type ipv4_addr; flags dynamic; size 65536; }
set held80_6 { type ipv6_addr; flags dynamic; size 65536; }
set rate80_4 { type ipv4_addr; flags dynamic,timeout; timeout 1m; size 65536; }
set rate80_6 { type ipv6_addr; flags dynamic,timeout; timeout 1m; size 65536; }
chain input {
type filter hook input priority filter - 10; policy accept;
ct state invalid drop
# the whole node: more new connections than the process can read is a flood, whoever sends it
tcp dport { 80, 443 } ct state new limit rate over 20000/second burst 40000 packets drop
# :443, per source
tcp dport 443 ct state new add @held4 { ip saddr ct count over 1024 } drop
tcp dport 443 ct state new add @held6 { ip6 saddr & ffff:ffff:ffff:ffff:: ct count over 1024 } drop
tcp dport 443 ct state new add @rate4 { ip saddr limit rate over 200/second burst 400 packets } drop
tcp dport 443 ct state new add @rate6 { ip6 saddr & ffff:ffff:ffff:ffff:: limit rate over 200/second burst 400 packets } drop
# :80, per source
tcp dport 80 ct state new add @held80_4 { ip saddr ct count over 32 } drop
tcp dport 80 ct state new add @held80_6 { ip6 saddr & ffff:ffff:ffff:ffff:: ct count over 32 } drop
tcp dport 80 ct state new add @rate80_4 { ip saddr limit rate over 20/second burst 40 packets } drop
tcp dport 80 ct state new add @rate80_6 { ip6 saddr & ffff:ffff:ffff:ffff:: limit rate over 20/second burst 40 packets } drop
}
}
files/tunnel/deploy/sysctl.conf 11 lines · 794 B
# Kernel settings for an ALL tunnel node (the design plan, section 5): install as /etc/sysctl.d/99-all-tunnel.conf, then `sysctl --system`.
#
# SYN cookies: a flood of half-open connections never fills the accept queue, and a client that completes the handshake still gets served.
net.ipv4.tcp_syncookies = 1
# the queues the node's listeners (Go's default backlog follows somaxconn) and the kernel's half-open table can hold
net.core.somaxconn = 4096
net.ipv4.tcp_max_syn_backlog = 8192
# a connection the peer abandoned in FIN-WAIT is let go sooner (the node's own idle limits and TCP keepalive end the rest)
net.ipv4.tcp_fin_timeout = 20
# connection tracking must hold what nftables counts: the table is the size of the node's busiest day, with room
net.netfilter.nf_conntrack_max = 262144
files/tunnel/doc.go 171 lines · 15.8 KB
// Command all-tunnel is the SNI-routing tunnel node of ALL 2.0 end-to-end encryption (the design plan, sections 2, 3 and 5, and
// the contract document, which this program implements).
//
// What it does: ChatGPT (any HTTPS client) connects to :443 and speaks normal TLS. The node reads only the cleartext ClientHello to learn the SNI name,
// finds the device that owns that name, and copies raw bytes between the client and the device. The TLS server is the DEVICE (it holds the private
// key and the publicly trusted certificate for its own name); this program never decrypts, never terminates and never rewrites a client's TLS, and it
// never logs payload bytes, addresses or names (only times, sizes, reasons and stream ids).
//
// # Ports
//
// There are exactly two, and no other:
//
// :443 the one public listener. The first thing read is the ClientHello's SNI:
// - the node's OWN name (-node-host): the control handler. The device speaks TLS 1.3 to it, with the node's public-CA certificate
// and ALPN all-tunnel/2, and protocol v2 inside (control and data connections, below);
// - any other name: a client of a device. If a device is online for that name and may serve, the node sends it OPEN and splices the
// client's bytes to the data connection that comes back; otherwise the client gets a TLS alert and nothing else happens.
// :80 the node's own ACME webroot and nothing else: GET/HEAD /.well-known/acme-challenge/<token> for the node's own name, from
// -acme-webroot. It never reaches a device (a certificate for a device is issued over DNS-01, and CAA allows nothing else).
//
// The 1.1 experiment's separate device port and the self-signed certificate that devices pinned are gone.
//
// # Device protocol (version 2)
//
// Every connection to the node's own name first completes TLS, then the device sends ONE text line (ASCII 0x20-0x7e, fields separated by one space,
// ended by LF, at most 8192 bytes, within first_line_timeout_s). The grammar is contracts/schemas/wire-v2.grammar.json; proto.go implements it and
// protocol_test.go runs every golden vector through it. The first line tells the two kinds of connection apart.
//
// Control connection (one per device, kept open):
//
// device -> node HELLO 2 <session> the device's session JWT (aud all-tunnel, 300 s); no host name
// node -> device CHALLENGE <nonce> once the session verified OFFLINE against the control plane's keys
// device -> node PROOF <dpop> a DPoP proof over the NODE's URL (htu), the nonce and the session, signed by the session's bound key
// node -> device OK <json> after the node asked the control plane (authorize): host, status, valid_until, windows, relayed as given
// or ERR <code> a refused HELLO closes the connection
//
// The host a device is served under comes ONLY from the control plane's answer; nothing the device says can name one. The connection lives until the
// answer's valid_until: before that moment minus renew_before_valid_until_s the device sends
//
// device -> node RENEW <new session> the same subject; CHALLENGE and PROOF follow, then the node calls authorize AGAIN and answers
// node -> device OK <json> with the NEW valid_until and status (a trial that ended is gated from here, with no event needed)
//
// and a RENEW that is refused keeps the connection with its old valid_until. Without a RENEW the node sends CLOSE expired at valid_until and closes. A
// second authenticated HELLO of the same subject replaces the first (CLOSE replaced; its open streams keep running); a HELLO whose answer names a host
// that another subject holds is refused, and a RENEW whose answer names another host closes the connection (CLOSE revoked): the old host stops being
// served at once. Status paused keeps the connection but opens no stream; gated opens them (held to the day's byte and stream budget, see
// Hardening). PING is answered with PONG; a connection silent for dead_peer_s is dropped (CLOSE idle). HINT carries source addresses the device
// marked good (the node's hint table, and Config.OnHint).
//
// OPEN:
//
// node -> device OPEN <stream-id> <attach-secret> <peer-ip> <class>
//
// is sent ONLY after the client's ClientHello is complete and well formed (a truncated, stalled, oversized or malformed one, or one without a usable
// SNI, never produces an OPEN), never for a paused device, never at or after valid_until. stream-id is 128 random bits, attach-secret is 256 random
// bits and valid for ONE ATTACH, peer-ip is the client's address (no zone) and class is openai (the address is in OpenAI's published list), hint (the device marked it with
// HINT) or anon (Config.Classify overrides this).
//
// Data connection (one per OPEN, dialed by the device to the same name and port):
//
// device -> node ATTACH <stream-id> <attach-secret>
// node -> device ERR auth | ERR unknown_stream | ERR bad_attach (then close), or NOTHING:
//
// on success there is NO reply line. The very next byte the node sends is the first byte of the raw stream of ONE client connection (the ClientHello,
// replayed exactly as received), and every byte the device sends from then on goes to that client. A wrong secret is refused and does not consume the
// stream; a used or unknown id is unknown_stream. EOF in one direction is propagated as a half-close and the other direction then has half_close_s to
// finish; each stream is bounded by the idle timeout, the maximum age and a byte cap, and streams per host and in total are capped.
//
// # Trust and what the node knows
//
// The node trusts the control plane's keys (a KeySource) to verify sessions and the control plane's answer (a ControlPlane) to decide what a device
// may do. The production build wires the real ones (backend_prod.go); the test-node build wires fakes. Every number, window and pattern is read from
// policy.json (a byte-identical copy of contracts/policy.json that the contract generator's sync writes).
//
// # The control plane (controlplane.go, jwks.go, feed.go)
//
// The node holds ONE secret: its own Ed25519 signing key (-node-key-file, mode 0600; -node-key-init makes it and prints the TUNNEL_NODE_KEYS entry that
// registers it, removing the kid from that variable revokes the node). Every request to /v1/tunnel/* carries an assertion in x-all-node-assertion, a
// 60 s JWS bound to the method, the exact URL and the SHA-256 of the exact body, used once. Nothing else authenticates the node: no shared secret, no
// commerce key. The client reads only the configured control plane (never a redirect, never a URL an answer names), caps every answer, and turns
// whatever it cannot read into bad_response: no text from the network reaches a device or a log.
//
// authorize asked at every HELLO and at every RENEW (the end of a trial turns a connection into gated with no event); at most
// authorization.authorize_max_concurrent calls run at once and what waits is bounded, the rest is told control_unavailable
// JWKS the published keys: fetched at start and every cache_s, reloaded when a session names an unknown kid (one fetch per minimum gap,
// however many sessions ask), replaced whole on every fetch, at most max_keys; after stale_fail_closed_s without a refresh the node
// vouches for nothing: new HELLOs and RENEWs are answered control_unavailable (not auth), connections already up are untouched
// feed GET /v1/tunnel/feed with a held poll (wait), applied within timing.feed_apply_deadline_s: revoke, release and license_end close the
// connection (CLOSE revoked/released) and kill the streams of the host; pause kills the streams, stops new ones and forwards the
// device's signed statement; resume asks the control plane again (its answer decides); kill/kill_clear are the global switch; reset
// closes everything so that devices come back and are authorized again. Events are applied once, in epoch order; a duplicate, a
// replay or an event older than a connection's last answer changes nothing
// STATEMENT the access statement of an authorize answer, forwarded unchanged right after the OK (under the write lock: no OPEN comes before it)
//
// # When the control plane does not answer
//
// What is authenticated goes on until valid_until: nothing is asked of the control plane to keep a connection or to open a stream. A new HELLO needs
// a good authorize. A PAID connection whose valid_until arrives while the control plane is not answering (its own RENEW got a 5xx, a 429 or a
// timeout, or nothing has answered for two poll periods) is kept for authorization.grace_paid_s more, opens no new stream in that time, and is closed
// when it is over; a trial or a gated connection is closed at valid_until, and so is a paid one whose RENEW the control plane answered (it said no).
// A connection that ends because its grant ended (expired, revoked, released, killed) takes the streams of its host with it.
//
// # Draining (drain.go)
//
// SIGTERM stops new ClientHellos (a TLS alert, no OPEN) and new HELLOs (ERR control_unavailable), tells every device DRAIN <quiet_s> (prefer the other
// node), keeps serving the streams that are open, the ATTACH of streams that already have their OPEN and the RENEW of the devices that hold them, and
// exits when the streams are gone or timing.drain_quiet_s (20 minutes, -drain-quiet) is over, whichever comes first, with CLOSE drain to the devices.
// A second SIGTERM stops waiting; SIGINT stops at once; SIGHUP reads the node's certificate again (certreload.go also looks at the files on its own).
//
// # Hardening (limits.go, hints.go, gated.go, openai_prefixes.go, observer.go, metrics.go; deploy/)
//
// Every number is policy.node_limits or policy.gated; nothing in this program types one. A SOURCE is an IPv4 address, or the /64 of an IPv6 one.
//
// at accept a source that is banned, that opens connections faster than per_ip_new_per_s, or that holds per_ip_concurrent already (pre-hello, and
// the client streams it has open: a device's control connection stops counting when it authenticates, a data connection when it
// attaches) is dropped with no answer; an address of the OpenAI list has openai_per_ip_factor times the limits and is never banned;
// an address that holds an authenticated control connection has device_per_ip_factor times the limits (a device opens one data
// connection per stream it serves), and a connection that has authenticated or attached gives back the token it took from the rate
// bucket; :80 is under the same limits
// ClientHello 5 s (client_hello_deadline_s), no OPEN before it is complete and well formed
// unknown names a source that tries more than unknown_sni_per_ip DISTINCT names that are not a device online (or sends no TLS) within
// unknown_sni_window_s is banned for unknown_sni_ban_s, in memory; the same name again costs nothing (ChatGPT retrying an offline device
// is not a scan); an address a device marked with HINT is never struck, and the shared overflow entry (what sources get when the
// table is full) is never struck or banned, so a scanner cannot turn a full table into a ban of every new source
// per device label_streams_max streams: label_anon_slice for any source, and the rest only for sources the device marked with HINT (a marked source
// may also use free anonymous room); new streams from sources that are not marked are paced at label_new_per_s (burst label_new_burst)
// global global_streams_max, of which openai_reserved_pct percent only openai or hint sources may use (a quota, not an allowlist: the class
// comes from OpenAI's published list, read at start and every openai_prefix_refresh_s, kept in a file, and a list that is empty, corrupt,
// too big or has a prefix wider than the policy's minimum replaces nothing)
// HINT addresses a device says just completed authenticated calls: memory only, hint_ttl_s from the last HINT, at most
// hint_addresses_per_label per device, filed under the host the control plane gave THAT connection
// gated gated.bytes_per_day and gated.streams_per_day per device host and UTC day, in memory; counted on every status the control plane gives
// (authorize at HELLO and RENEW, and the refresh after a pause or resume), so a device that turns gated mid-connection is cut on its
// open streams when the bytes run out, and reconnecting does not buy a new budget. The budget is spent by every stream of the host,
// whoever opens it: a device's name is public (certificate transparency), so a third party can use up a gated device's day (200
// streams or 10 MB) and keep its paywall card from reaching the user until midnight UTC. Accepted: a gated device is not a service
// anyone depends on, and the price of the alternative (telling clients apart) is a rule that fails open
// sockets TCP keepalive from tcp_keepalive_s on everything the node accepts; idle and maximum age of a stream are timing.stream_idle_s and
// timing.stream_max_age_s
// logs no address, no name, no label, no device id in a log line, ever: a line that must tell two sources apart carries Server.short, the
// per-boot salted short hash; lines a flood can repeat are written at most once a second per reason, with how many were left out
// observer the cleartext start of a handshake, counted by a closed list of words: the groups a ClientHello offers (X25519MLKEM768 included) and
// whether it carries an ECH extension, the version, cipher and group the device's ServerHello selects, HelloRetryRequests. Nothing
// else, nothing that names a client
// /metrics -metrics-addr (loopback only, 127.0.0.1:9101 in the production build) serves /metrics and /healthz: aggregate counters, no series
// per device, label, address or name, the node certificate's not_after and the age of the control plane's keys included; a metrics
// port that cannot be bound is a log line (metrics_unavailable), not a node that does not start
// kernel deploy/nftables.conf and deploy/sysctl.conf: per-source connection and rate limits above the node's own, SYN cookies. The kernel
// counts every connection a source HOLDS, a device's data connections (one per open stream) included, while the node frees its own
// slot at attach: the held limit is sized for that
//
// # Test node
//
// `go build -tags testnode` (build.sh test-node) builds the same node with file-driven fakes for the keys and the control plane. It is what
// ALL_TEST_NODE_BIN names for the device's tests; see testnode.go. It is never deployed.
// With -test-tap FILE it also records every byte it forwards and every control line for the blindness gate (tap_test_hook.go); the build with
// `-tags "testnode blindmutant"` (build.sh blind-mutant) is the node that gate must catch (blindmutant_test_hook.go). Neither is ever deployed.
//
// # Run
//
// all-tunnel -node-id t1 -node-kid node-t1-1 -node-host t1.nodes.example.org -node-key-file node.key \
// -cert-file fullchain.pem -key-file privkey.pem -acme-webroot /var/lib/all-tunnel/acme
//
// See main.go and backend_prod.go for every flag; each also has a TUNNEL_* environment variable.
//
// # Operating it
//
// deploy/ holds what a node needs around the binary (the unit, the certbot units and hooks, the volatile journal, nftables, sysctl). the deploy tool
// installs it and runs the whole procedure (provision, bootstrap, canary-first deploy with its DNS, stream and health gates, rollback); the canary tool is
// the canary device the monitors probe through each node; the node runbook is the procedure, drain and lost-call caveat included.
package main
files/tunnel/drain.go 64 lines · 2.7 KB
package main
import (
"time"
)
// Draining (the design plan, section 5): what a node does on SIGTERM, so that a deploy or a restart costs the people on it as little as
// possible. In this order:
//
// 1. It stops taking anything new. A client that connects now gets a TLS alert and no OPEN (the DNS no longer points at a node being taken down, but
// a client that resolved it a minute ago still arrives); a device that sends a HELLO gets ERR control_unavailable and goes to the other node.
// The listeners stay open on purpose: a device's ATTACH for a stream that already has its OPEN must still be served, and closing the port would
// turn a drain into a reset for every client that is a handshake away from its stream.
// 2. It tells every device it serves DRAIN <quiet_s>: prefer the other node for new connections; the open streams continue here for the quiet period.
// 3. It waits until the streams are gone or the quiet period (policy timing.drain_quiet_s, 20 minutes) is over, whichever comes first, then
// closes what is left and exits. A call of 900 s that started before SIGTERM finishes; one that would run past the quiet period is cut, and the
// runbook says so (it may already have run on the computer).
//
// The control connections stay up for the whole drain (a RENEW is answered), and the feed keeps being applied: a revoke during a drain still cuts.
// drainPoll is how often a drain looks at whether the streams are gone.
const drainPoll = 250 * time.Millisecond
// Draining reports whether the node is draining.
func (s *Server) Draining() bool { return s.draining.Load() }
// Drain runs a drain to its end (the streams gone, or quiet over) and shuts the node down. It returns when the node is down. A second call is a no-op.
func (s *Server) Drain(quiet time.Duration) {
s.drainFor.CompareAndSwap(0, int64(quiet))
if !s.draining.CompareAndSwap(false, true) {
return
}
s.log.Info("draining", "quiet_s", int64(quiet/time.Second), "streams", s.streams.Load())
line := drainLine(quiet)
for _, cc := range s.routes.controls("") {
if cc.isReady() {
_ = cc.write(line) // a device that cannot be told is no worse off: it learns when the connection closes
}
}
over := make(chan struct{})
stop := s.clk.AfterFunc(quiet, func() { close(over) })
defer stop()
for s.streams.Load() > 0 || s.pendingCount() > 0 {
select {
case <-over:
s.log.Warn("drain_quiet_over", "streams", s.streams.Load())
s.shutdown(0, closeDrain)
return
case <-s.ctx.Done():
return
default:
}
time.Sleep(drainPoll)
}
s.log.Info("drained")
s.shutdown(s.cfg.ShutdownGrace, closeDrain)
}
func (s *Server) pendingCount() int {
s.mu.Lock()
defer s.mu.Unlock()
return len(s.pending)
}
files/tunnel/drain_test.go 424 lines · 13.7 KB
package main
import (
"encoding/json"
"os"
"path/filepath"
"strconv"
"strings"
"sync/atomic"
"syscall"
"testing"
"time"
)
// SIGTERM drains (the design plan, section 5): nothing new is accepted, what is open finishes, the devices are told to prefer the other node,
// and the node is gone by the end of the quiet period.
func drainAsync(e *testEnv, quiet time.Duration) (done chan struct{}, started time.Time) {
done = make(chan struct{})
started = time.Now()
go func() {
e.srv.Drain(quiet)
close(done)
}()
return done, started
}
func TestDrain_NothingNewIsAcceptedOpenStreamsContinueDrainIsDeliveredAndTheNodeLeavesWhenTheyEnd(t *testing.T) {
e := newTestEnv(t, nil)
d, other := e.newDevice(), e.newDevice()
dc := e.connectOK(d)
ds := dc.serve(nil)
oc := e.connectOK(other)
otherServer := oc.serve(nil)
c, err := e.clientTLS(d)
if err != nil {
t.Fatal(err)
}
if resp, err := postOver(c, d.host, []byte("before")); err != nil || string(resp) != "echo:before" {
t.Fatalf("%q %v", resp, err)
}
quiet := 8 * time.Second
done, started := drainAsync(e, quiet)
for name, dev := range map[string]*devServer{"first": ds, "second": otherServer} {
dev := dev
eventually(t, name+" told DRAIN", 3*time.Second, func() bool { return containsLine(dev.otherLines(), "DRAIN "+strconv.Itoa(int(quiet/time.Second))) })
}
// no new client: the TLS alert, no OPEN, nothing forwarded
opens := ds.openCount()
if _, err := e.clientTLS(d); err == nil {
t.Fatal("a client was accepted by a draining node")
}
if _, err := e.clientTLS(other); err == nil {
t.Fatal("a client of another device was accepted by a draining node")
}
time.Sleep(100 * time.Millisecond)
if ds.openCount() != opens || otherServer.openCount() != 0 {
t.Fatal("a draining node sent an OPEN")
}
// no new device: told to go to the other node
stranger := e.newDevice()
sc := e.connect(stranger)
if line := sc.hello(func() string { return e.session(stranger, nil) }, func(s, n string) string { return e.proof(stranger, s, n, nil) }); line != "ERR control_unavailable" {
t.Fatalf("a HELLO during a drain: %q", line)
}
// the stream that was open goes on, and so does a RENEW of the device that holds it
if resp, err := postOver(c, d.host, []byte("during")); err != nil || string(resp) != "echo:during" {
t.Fatalf("an open stream during a drain: %q %v", resp, err)
}
dc.drainFeed() // the DRAIN line the device was sent is still unread
if r := dc.renew(); !strings.HasPrefix(r, "OK ") {
t.Fatalf("renew during a drain: %q", r)
}
select {
case <-done:
t.Fatal("the node left with a stream open")
case <-time.After(300 * time.Millisecond):
}
// the last stream ends: the node closes the devices (CLOSE drain) and exits well inside the quiet period
_ = c.Close()
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("the node did not leave when its last stream ended")
}
if took := time.Since(started); took >= quiet {
t.Fatalf("it took %s, the quiet period is %s", took, quiet)
}
<-ds.done
if !containsLine(ds.otherLines(), "CLOSE drain") {
t.Fatalf("lines %q", ds.otherLines())
}
if _, err := e.dialNode(); err == nil {
t.Fatal("the node still listens after it drained")
}
}
func TestDrain_ALongStreamIsCutAtTheEndOfTheQuietPeriodAndNotBefore(t *testing.T) {
e := newTestEnv(t, nil)
d := e.newDevice()
dc := e.connectOK(d)
_ = dc.serve(nil)
stream := streamOpen(t, e, d)
quiet := 2 * time.Second
done, started := drainAsync(e, quiet)
time.Sleep(quiet / 2)
if stream.dead(50 * time.Millisecond) {
t.Fatal("the stream was cut before the quiet period was over")
}
select {
case <-done:
case <-time.After(quiet + 4*time.Second):
t.Fatal("the node did not exit by the end of the quiet period")
}
if took := time.Since(started); took < quiet-100*time.Millisecond || took > quiet+3*time.Second {
t.Fatalf("it took %s for a quiet period of %s", took, quiet)
}
if !stream.dead(2 * time.Second) {
t.Fatal("the stream outlived the node")
}
}
func TestDrain_AnAttachForAStreamThatAlreadyHasItsOpenIsStillServed(t *testing.T) {
e := newTestEnv(t, nil)
d := e.newDevice()
dc := e.connectOK(d)
ds := dc.serve(nil)
ds.noAttach = true // the device is slow: it has been sent the OPEN and has not attached yet
type result struct {
resp []byte
err error
}
got := make(chan result, 1)
go func() {
c, err := e.clientTLS(d)
if err != nil {
got <- result{nil, err}
return
}
defer c.Close()
resp, err := postOver(c, d.host, []byte("late"))
got <- result{resp, err}
}()
eventually(t, "the OPEN", 3*time.Second, func() bool { return ds.openCount() == 1 })
done, _ := drainAsync(e, 10*time.Second)
eventually(t, "draining", 2*time.Second, func() bool { return e.srv.Draining() })
open := ds.openList()[0]
ds.wg.Add(1)
go ds.attach(open.str("stream_id"), open.str("secret"))
select {
case r := <-got:
if r.err != nil || string(r.resp) != "echo:late" {
t.Fatalf("a client whose stream was being set up when the drain began: %q %v", r.resp, r.err)
}
case <-time.After(5 * time.Second):
t.Fatal("the stream that was being set up was not served")
}
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("no exit after the last stream")
}
}
func TestDrain_ADeviceThatIsAuthenticatingWhenTheDrainStartsIsToldToo(t *testing.T) {
// A HELLO past the gate whose authorize is still running when SIGTERM comes: its OK is followed by DRAIN, so it never serves as if nothing happened.
e := newTestEnv(t, nil)
holder := e.newDevice()
hc := e.connectOK(holder)
_ = hc.serve(nil)
held := streamOpen(t, e, holder) // a stream that keeps the node draining for a while
defer held.Close()
d := e.newDevice()
e.plan(d).delay = 400 * time.Millisecond
dc := e.connect(d)
dc.session = e.session(d, nil)
dc.send("HELLO 2 " + dc.session)
m, err := parseLine(dc.recv(), dirN2D)
if err != nil || m.str("verb") != "CHALLENGE" {
t.Fatal("no challenge")
}
dc.send("PROOF " + e.proof(d, dc.session, m.str("nonce"), nil))
time.Sleep(100 * time.Millisecond) // inside the slow authorize
done, _ := drainAsync(e, 5*time.Second)
eventually(t, "draining", 2*time.Second, func() bool { return e.srv.Draining() })
if line := dc.recv(); !strings.HasPrefix(line, "OK ") {
t.Fatalf("%q", line)
}
if line := dc.recv(); line != "DRAIN 5" {
t.Fatalf("after the OK of a device that authenticated during the drain: %q", line)
}
_ = dc.c.Close()
_ = held.Close()
<-done
}
func TestDrain_ARevokeInTheMiddleOfADrainStillCuts(t *testing.T) {
f := &scriptedFeed{}
e := withFeed(t, f)
d := e.newDevice()
dc := e.connectOK(d)
ds := dc.serve(nil)
stream := streamOpen(t, e, d)
done, _ := drainAsync(e, 6*time.Second)
eventually(t, "draining", 2*time.Second, func() bool { return e.srv.Draining() })
f.push(e.event(feedRevoke, d))
if !stream.dead(3 * time.Second) {
t.Fatal("a revoke during a drain did not cut the stream")
}
eventually(t, "CLOSE revoked", 2*time.Second, func() bool { return containsLine(ds.otherLines(), "CLOSE revoked") })
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("the node did not leave once nothing was open")
}
}
func TestDrain_ASecondDrainIsNoOp(t *testing.T) {
e := newTestEnv(t, nil)
done, _ := drainAsync(e, time.Second)
<-done
finished := make(chan struct{})
go func() { e.srv.Drain(time.Second); close(finished) }()
select {
case <-finished:
case <-time.After(2 * time.Second):
t.Fatal("a second Drain blocked")
}
}
// A call of 900 s that started before SIGTERM finishes; the quiet period is the policy's twenty minutes, on a clock the test moves.
func TestDrain_ASigtermDuringA900SecondStreamLetsItFinishAndOneThatNeverEndsIsCutAtTwentyMinutes(t *testing.T) {
if policyData.Timing.DrainQuietS != 1200 {
t.Fatalf("the plan says 20 minutes, the policy says %d s", policyData.Timing.DrainQuietS)
}
for _, endsAt := range []time.Duration{900 * time.Second, 0} {
e := newClockEnv(t, nil)
d := e.newDevice()
dc := e.connectOK(d)
ds := dc.serve(nil)
stream := streamOpen(t, e, d)
var exited atomic.Bool
done := make(chan struct{})
go func() {
e.srv.Drain(e.srv.cfg.DrainQuiet)
exited.Store(true)
close(done)
}()
eventually(t, "DRAIN delivered", 3*time.Second, func() bool { return containsLine(ds.otherLines(), "DRAIN 1200") })
e.clk.Advance(899 * time.Second)
time.Sleep(400 * time.Millisecond)
if exited.Load() || stream.dead(50*time.Millisecond) {
t.Fatalf("ends at %s: the stream or the node was cut at 899 s", endsAt)
}
if endsAt > 0 {
// the call finishes at 900 s: the node leaves at once, 300 s early
e.clk.Advance(time.Second)
_ = stream.Close()
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("the node did not leave after the 900 s stream finished")
}
continue
}
e.clk.Advance(300 * time.Second) // 1199 s
time.Sleep(400 * time.Millisecond)
if exited.Load() {
t.Fatal("the node left before the quiet period was over")
}
e.clk.Advance(2 * time.Second) // 1201 s
select {
case <-done:
case <-time.After(6 * time.Second):
t.Fatal("a stream that never ends kept the node past the quiet period")
}
if !stream.dead(2 * time.Second) {
t.Fatal("the stream outlived the node")
}
}
}
// ---------------------------------------------------------------------------------------------------------------------------- the real process
func TestTestNodeBinary_SigtermDrainsAndTheProcessExitsInsideTheQuietPeriod(t *testing.T) {
for _, longStream := range []bool{false, true} {
p := launchTestNode(t, 600, "-drain-quiet", "3s")
e, d, cmd := p.e, p.d, p.cmd
dc := e.connectOK(d)
ds := dc.serve(nil)
stream := streamOpen(t, e, d)
if !longStream {
_ = stream.Close()
}
start := time.Now()
if err := cmd.Process.Signal(syscall.SIGTERM); err != nil {
t.Fatal(err)
}
eventually(t, "DRAIN 3 delivered", 3*time.Second, func() bool { return containsLine(ds.otherLines(), "DRAIN 3") })
if longStream {
// no new client; the open one goes on
if _, err := e.clientTLS(d); err == nil {
t.Fatal("a client was accepted by a draining process")
}
if resp, err := postOver(stream.Conn, d.host, []byte("still")); err != nil || string(resp) != "echo:still" {
t.Fatalf("the open stream during the drain: %q %v", resp, err)
}
}
select {
case err := <-p.exited:
if err != nil {
t.Fatalf("exit: %v\n%s", err, p.logs.String())
}
case <-time.After(10 * time.Second):
t.Fatalf("the process did not exit (long stream: %v)\n%s", longStream, p.logs.String())
}
took := time.Since(start)
switch {
case !longStream && took > 2*time.Second:
t.Errorf("an idle node took %s to drain", took)
case longStream && (took < 2500*time.Millisecond || took > 3*time.Second+3*time.Second):
t.Errorf("a node with a stream that never ends left after %s; the quiet period is 3 s", took)
}
<-ds.done
if !containsLine(ds.otherLines(), "CLOSE drain") {
t.Errorf("lines %q", ds.otherLines())
}
}
}
func TestTestNodeBinary_ASecondSigtermDuringADrainStopsWaiting(t *testing.T) {
p := launchTestNode(t, 600, "-drain-quiet", "5m")
dc := p.e.connectOK(p.d)
_ = dc.serve(nil)
stream := streamOpen(t, p.e, p.d)
defer stream.Close()
if err := p.cmd.Process.Signal(syscall.SIGTERM); err != nil {
t.Fatal(err)
}
time.Sleep(500 * time.Millisecond)
select {
case <-p.exited:
t.Fatal("the process left with a stream open and five minutes of quiet period left")
default:
}
if err := p.cmd.Process.Signal(syscall.SIGTERM); err != nil {
t.Fatal(err)
}
select {
case err := <-p.exited:
if err != nil {
t.Fatalf("exit: %v", err)
}
case <-time.After(5 * time.Second):
t.Fatalf("a second SIGTERM did not stop the drain\n%s", p.logs.String())
}
}
func TestTestNodeBinary_AFeedFileRevokesAStreamAndTheKillSwitchClosesEverything(t *testing.T) {
dir := t.TempDir()
feed := filepath.Join(dir, "feed.json")
writeFeed := func(epoch int, events ...map[string]any) {
t.Helper()
list := []any{}
for _, ev := range events {
list = append(list, ev)
}
b, _ := json.Marshal(map[string]any{"epoch": epoch, "events": list})
if err := os.WriteFile(feed+".tmp", b, 0o600); err != nil {
t.Fatal(err)
}
if err := os.Rename(feed+".tmp", feed); err != nil {
t.Fatal(err)
}
}
writeFeed(0)
p := launchTestNode(t, 600, "-test-feed", feed, "-feed-poll", "100ms")
e, d := p.e, p.d
dc := e.connectOK(d)
ds := dc.serve(nil)
stream := streamOpen(t, e, d)
at := time.Now().Unix()
writeFeed(1, map[string]any{"epoch": 1, "type": "revoke", "device_id": d.id, "label": d.label, "at": at})
eventually(t, "CLOSE revoked", 5*time.Second, func() bool { return containsLine(ds.otherLines(), "CLOSE revoked") })
if !stream.dead(3 * time.Second) {
t.Fatal("the revoked device's stream is still open")
}
// the kill switch: nobody is served
writeFeed(2, map[string]any{"epoch": 1, "type": "revoke", "device_id": d.id, "label": d.label, "at": at},
map[string]any{"epoch": 2, "type": "kill", "device_id": nil, "label": nil, "at": at})
time.Sleep(600 * time.Millisecond)
c := e.connect(d)
if line := c.hello(func() string { return e.session(d, nil) }, func(s, n string) string { return e.proof(d, s, n, nil) }); line != "ERR control_unavailable" {
t.Fatalf("a HELLO while the kill switch is on: %q", line)
}
writeFeed(3, map[string]any{"epoch": 3, "type": "kill_clear", "device_id": nil, "label": nil, "at": at})
eventually(t, "served again", 5*time.Second, func() bool {
cc := e.connect(d)
return strings.HasPrefix(cc.hello(func() string { return e.session(d, nil) }, func(s, n string) string { return e.proof(d, s, n, nil) }), "OK ")
})
if strings.Contains(p.logs.String(), d.id) || strings.Contains(p.logs.String(), d.label) {
t.Fatal("the process logs name a device")
}
}
// drainFeed discards what the node already sent and the test did not read.
func (dc *devConn) drainFeed() {
for {
select {
case <-dc.feed:
default:
return
}
}
}
files/tunnel/fakecp_test.go 324 lines · 9.7 KB
package main
import (
"crypto/ed25519"
"crypto/rand"
"io"
"net/http"
"net/http/httptest"
"strconv"
"sync"
"testing"
"time"
)
// fakeCP is a control plane over real HTTPS (httptest): it authenticates the node with the contracts' own verifier (the Go twin of the TypeScript one
// the Worker runs), decides authorize with the environment's fakeControl, keeps a feed and publishes the keys. It is what the real client, the JWKS
// cache and the feed loop are tested against.
type fakeCP struct {
t *testing.T
env *testEnv
srv *httptest.Server
nodeKID string
nodeKey ed25519.PrivateKey
mu sync.Mutex
nodeKeys map[string]ctNodeKey
seen map[string]bool
jwks map[string]ed25519.PublicKey // kid -> key, what GET /.well-known/device-jwks.json says
down int // when not 0, every endpoint answers with this status
hold time.Duration // how long a feed poll with wait is held, in real time, when nothing is new
events []FeedEvent
head int64
floor int64
changed chan struct{}
bodies [][]byte // the body of every authenticated authorize
headers []http.Header // the headers of every authenticated request, to see that nothing but the assertion authenticates
refused int // requests whose assertion was refused
feedLog []feedCall
counts map[string]int
inflight int
maxIn int
delay time.Duration
// redirect, when set, answers authorize with a redirect to that URL
redirect string
}
type feedCall struct {
since int64
wait int64
url string
}
func newFakeCP(t *testing.T, env *testEnv) *fakeCP {
t.Helper()
pub, priv, err := ed25519.GenerateKey(rand.Reader)
if err != nil {
t.Fatal(err)
}
cp := &fakeCP{
t: t, env: env, nodeKID: "node-t1-1", nodeKey: priv,
nodeKeys: map[string]ctNodeKey{}, seen: map[string]bool{}, jwks: map[string]ed25519.PublicKey{},
changed: make(chan struct{}), counts: map[string]int{},
}
cp.nodeKeys[cp.nodeKID] = ctNodeKey{node: testNodeID, x: b64Encode(pub)}
cp.jwks[testKID] = env.cpKey.Public().(ed25519.PublicKey)
cp.srv = httptest.NewTLSServer(http.HandlerFunc(cp.handle))
t.Cleanup(cp.srv.Close)
return cp
}
func (cp *fakeCP) url() string { return cp.srv.URL }
// control is the real client of the node, pointed at this control plane.
func (cp *fakeCP) control() *HTTPControl {
hc, err := NewHTTPControl(HTTPControlConfig{BaseURL: cp.srv.URL, NodeID: testNodeID, KID: cp.nodeKID, Key: cp.nodeKey, Client: cp.srv.Client(), Now: cp.env.now, Version: "test"})
if err != nil {
cp.t.Fatal(err)
}
return hc
}
func (cp *fakeCP) setDown(status int) {
cp.mu.Lock()
cp.down = status
cp.mu.Unlock()
}
// addEvent appends a feed event (epoch is the next one) and wakes the polls that are held.
func (cp *fakeCP) addEvent(kind string, d *testDevice) int64 {
cp.mu.Lock()
defer cp.mu.Unlock()
cp.head++
e := FeedEvent{Epoch: cp.head, Type: kind, At: cp.env.now().Unix()}
if d != nil {
e.DeviceID, e.Label = d.id, d.label
}
cp.events = append(cp.events, e)
close(cp.changed)
cp.changed = make(chan struct{})
return e.Epoch
}
func (cp *fakeCP) callCount(name string) int {
cp.mu.Lock()
defer cp.mu.Unlock()
return cp.counts[name]
}
func (cp *fakeCP) maxInflight() int {
cp.mu.Lock()
defer cp.mu.Unlock()
return cp.maxIn
}
func (cp *fakeCP) feedCalls() []feedCall {
cp.mu.Lock()
defer cp.mu.Unlock()
return append([]feedCall(nil), cp.feedLog...)
}
func (cp *fakeCP) reply(w http.ResponseWriter, status int, body map[string]any) {
w.Header().Set("content-type", "application/json")
w.WriteHeader(status)
_, _ = io.WriteString(w, canonicalJSON(body))
}
func (cp *fakeCP) refuse(w http.ResponseWriter, status int, word string, extra map[string]any) {
body := map[string]any{"ok": false, "error": word}
for k, v := range extra {
body[k] = v
}
cp.reply(w, status, body)
}
func (cp *fakeCP) handle(w http.ResponseWriter, r *http.Request) {
cp.mu.Lock()
down := cp.down
cp.counts[r.URL.Path]++
cp.mu.Unlock()
if down != 0 {
cp.refuse(w, down, "end_to_end_unavailable", map[string]any{"retry_after_s": 15})
return
}
switch {
case r.URL.Path == jwksPath && r.Method == http.MethodGet:
cp.serveJWKS(w)
case r.URL.Path == authorizePath && r.Method == http.MethodPost:
cp.serveAuthorize(w, r)
case r.URL.Path == feedPath && r.Method == http.MethodGet:
cp.serveFeed(w, r)
default:
cp.refuse(w, 404, "not_found", nil)
}
}
func (cp *fakeCP) serveJWKS(w http.ResponseWriter) {
cp.mu.Lock()
list := []any{}
for kid, key := range cp.jwks {
list = append(list, map[string]any{"kty": "OKP", "crv": "Ed25519", "x": b64Encode(key), "kid": kid, "alg": "EdDSA", "use": "sig", "key_ops": []any{"verify"}})
}
cp.mu.Unlock()
cp.reply(w, 200, map[string]any{"keys": list})
}
// authenticate checks the node's assertion for exactly this request with the contracts' verifier. It answers 401 and reports false when it fails.
func (cp *fakeCP) authenticate(w http.ResponseWriter, r *http.Request, body []byte) bool {
cp.mu.Lock()
defer cp.mu.Unlock()
result := ctVerifyNodeAssertion(r.Header.Get("x-all-node-assertion"), epochSeconds(cp.env.now()), cp.nodeKeys, r.Method, "https://"+r.Host+r.URL.RequestURI(), body, 5, cp.seen)
if !result.ok {
cp.refused++
cp.reply(w, 401, map[string]any{"ok": false, "error": "unauthorized"})
return false
}
cp.seen[result.jti] = true
cp.headers = append(cp.headers, r.Header.Clone())
return true
}
func (cp *fakeCP) serveAuthorize(w http.ResponseWriter, r *http.Request) {
body, _ := io.ReadAll(io.LimitReader(r.Body, 1<<20))
if cp.redirect != "" {
http.Redirect(w, r, cp.redirect, http.StatusFound)
return
}
if !cp.authenticate(w, r, body) {
return
}
cp.mu.Lock()
cp.bodies = append(cp.bodies, body)
cp.mu.Unlock()
doc, err := parseJSON(string(body))
if err != nil || len(ctSchemaSet.validate("authorize-request.schema.json", doc)) > 0 {
cp.refuse(w, 400, "invalid_authorize_request", nil)
return
}
m := doc.(map[string]any)
req := AuthorizeRequest{DeviceID: m["device_id"].(string), SessionJTI: m["session_jti"].(string), Reason: m["reason"].(string), Node: m["node"].(string)}
cp.mu.Lock()
cp.inflight++
cp.maxIn = max(cp.maxIn, cp.inflight)
delay := cp.delay
cp.mu.Unlock()
defer func() {
cp.mu.Lock()
cp.inflight--
cp.mu.Unlock()
}()
if delay > 0 {
select {
case <-time.After(delay):
case <-r.Context().Done():
return
}
}
resp, aerr := cp.env.cp.Authorize(r.Context(), req)
if aerr != nil {
if ce, ok := aerr.(*ControlError); ok {
extra := map[string]any{}
if ce.Code == codeLicenseInactive {
extra["status"] = "trial_ended"
}
cp.refuse(w, ce.Status, ce.Code, extra)
return
}
cp.refuse(w, 503, "end_to_end_unavailable", nil)
return
}
cp.reply(w, 200, resp.toMap())
}
// replyFeed answers a feed poll after checking the answer against the contracts' own rules (schema, strictly increasing epochs, scopes): the fake must
// never be more lenient than the real control plane, or the node would be tested against something that does not exist.
func (cp *fakeCP) replyFeed(w http.ResponseWriter, since int64, body map[string]any) {
if doc, err := parseJSON(canonicalJSON(body)); err != nil {
cp.t.Errorf("the fake feed made a body that is not JSON: %v", err)
} else if _, reset := body["reset"]; !reset {
if errs := ctCheckFeed(doc, float64(since)); len(errs) > 0 {
cp.t.Errorf("the fake feed broke the contract: %v", errs)
}
}
cp.reply(w, 200, body)
}
func (cp *fakeCP) serveFeed(w http.ResponseWriter, r *http.Request) {
if !cp.authenticate(w, r, nil) {
return
}
since, err := strconv.ParseInt(r.URL.Query().Get("since"), 10, 64)
if err != nil {
cp.refuse(w, 400, "invalid_feed_request", nil)
return
}
wait, _ := strconv.ParseInt(r.URL.Query().Get("wait"), 10, 64)
cp.mu.Lock()
cp.feedLog = append(cp.feedLog, feedCall{since: since, wait: wait, url: r.URL.RequestURI()})
hold := cp.hold
cp.mu.Unlock()
deadline := time.Now().Add(min(hold, time.Duration(wait)*time.Second))
for {
cp.mu.Lock()
if since > cp.head || since < cp.floor {
head := cp.head
cp.mu.Unlock()
cp.replyFeed(w, since, map[string]any{"ok": true, "v": 1, "epoch": head, "server_time": cp.env.now().Unix(), "reset": true, "events": []any{}})
return
}
var out []any
for _, e := range cp.events {
if e.Epoch > since {
var device, label any
if !e.global() {
device, label = e.DeviceID, e.Label
}
out = append(out, map[string]any{"epoch": e.Epoch, "type": e.Type, "device_id": device, "label": label, "at": e.At})
}
}
changed, head := cp.changed, cp.head
cp.mu.Unlock()
if len(out) > 0 || !time.Now().Before(deadline) {
if out == nil {
out = []any{}
}
cp.replyFeed(w, since, map[string]any{"ok": true, "v": 1, "epoch": head, "server_time": cp.env.now().Unix(), "events": out})
return
}
select {
case <-changed:
case <-time.After(time.Until(deadline)):
case <-r.Context().Done():
return
}
}
}
// httpNode starts a node whose control plane, key set and feed are the real clients talking to a fakeCP over HTTPS, on the fake clock when clk is set.
func httpNode(t *testing.T, clk *fakeClock, mutate func(*Config)) (*testEnv, *fakeCP) {
t.Helper()
var cp *fakeCP
e := newEnv(t, clk, func(e *testEnv, cfg *Config) {
cp = newFakeCP(t, e)
control := cp.control()
health := &cpHealth{}
clock := newClock(e.now, cfg.AfterFunc)
jwks := NewJWKSCache(JWKSConfig{Fetch: control.FetchJWKS, Clock: clock, Log: cfg.Logger, MinReload: time.Millisecond, Observe: func(ok bool) {
if ok {
health.good(clock.Now())
} else {
health.bad(clock.Now())
}
}})
cfg.Keys, cfg.Control, cfg.Feed, cfg.Health = jwks, control, control, health
cfg.Background = append(cfg.Background, jwks.Run)
cfg.FeedStartWait = 5 * time.Second
cfg.MinRenewGap = 0
if mutate != nil {
mutate(cfg)
}
})
return e, cp
}
files/tunnel/feed.go 427 lines · 14.0 KB
package main
import (
"context"
"errors"
"sort"
"sync"
"time"
)
// The feed consumer (the design plan, sections 2, 3 and 5; the contract, section 4): what the node must stop serving, learned from
// GET /v1/tunnel/feed within policy timing.feed_apply_deadline_s. A node never decides to cut a device by itself; it cuts what the control plane says
// to cut, and it keeps serving what is already authenticated until valid_until when the control plane cannot be asked.
//
// revoke, release, license_end the device's control connection is closed and the streams of its host are killed
// pause no new stream; the streams in flight are killed; the connection stays (a resume arrives as a statement) and the
// node asks the control plane for the device's signed statement, which carries the pause, and forwards it
// resume the node asks again: the control plane's answer decides the status, and its statement (resumed) is forwarded
// kill, kill_clear the global switch: every connection is closed (CLOSE killed) and every stream killed, and nobody is served
// (control_unavailable) until it is cleared
// reset the cursor is not one the control plane can serve from: every tunnel is closed, the devices reconnect and are
// authorized again
//
// Events are applied in epoch order, each once: an event at or below the cursor is dropped, so a duplicate or a replay changes nothing, and a batch
// that arrives out of order is sorted first. An event older than a connection's last answer from the control plane (by the control plane's own clock)
// is already part of that answer and is skipped: after a restart the feed is replayed from the start, and a device that came back since its revoke
// must not be cut again by the old event.
// feedWaitSlack is how much longer than its hold the node lets one poll take.
const feedWaitSlack = 10 * time.Second
// startBackground starts the feed loop and the node's background tasks. The feed's first answer is awaited (bounded) so that the node does not accept
// a device before it knows what it must not serve.
func (s *Server) startBackground() {
for _, task := range append([]func(context.Context){s.openai.Run, s.sweepLoop}, s.cfg.Background...) {
s.bg.Add(1)
go func(task func(context.Context)) {
defer s.bg.Done()
task(s.ctx)
}(task)
}
if s.cfg.Feed == nil {
return
}
caughtUp := make(chan struct{})
s.bg.Add(1)
go func() {
defer s.bg.Done()
s.runFeed(s.ctx, caughtUp)
}()
select {
case <-caughtUp:
case <-time.After(s.cfg.FeedStartWait):
s.log.Warn("feed_late", "waited_ms", s.cfg.FeedStartWait.Milliseconds())
}
}
// runFeed polls until ctx ends. close(caughtUp) runs once, after the first answer applied or the first failure.
func (s *Server) runFeed(ctx context.Context, caughtUp chan struct{}) {
var cursor int64
failures := 0
first := true
signal := func() {
if first {
first = false
close(caughtUp)
}
}
defer signal()
for ctx.Err() == nil {
started := s.now()
wait := s.cfg.FeedPoll
if first {
wait = 0 // the first look answers at once: it is the one the listeners wait for
}
pctx, cancel := context.WithTimeout(ctx, s.cfg.FeedPoll+feedWaitSlack)
answer, err := s.cfg.Feed.Feed(pctx, cursor, wait)
cancel()
if err != nil {
if ctx.Err() != nil {
return
}
s.health.record(s.now(), err)
failures++
s.log.Warn("feed_failed", "reason", errorWord(err), "failures", failures)
signal()
// Retry quickly (never beyond the poll period: the moment the control plane is back, a revoke is heard within the apply deadline).
if !s.clk.sleep(ctx, jitter(min(backoff(failures), s.cfg.FeedPoll), 20)) {
return
}
continue
}
failures = 0
s.health.good(s.now())
next, applied := s.applyFeed(answer, cursor)
cursor = next
signal()
if !applied && wait > 0 {
// Nothing new in a poll that asked to be held. A control plane that holds it has already spent the period; one that answers at once is
// polled once per period. (The first look asks for no hold: the long polls start right after it.)
if rest := wait - s.now().Sub(started); rest > 0 && !s.clk.sleep(ctx, rest) {
return
}
}
}
}
// applyFeed applies one answer and returns the cursor for the next poll, and whether the answer carried anything the node acted on.
func (s *Server) applyFeed(answer FeedAnswer, cursor int64) (int64, bool) {
if answer.Skipped > 0 {
s.log.Warn("feed_events_skipped", "count", answer.Skipped)
}
if answer.Reset {
s.log.Warn("feed_reset", "cursor", cursor, "epoch", answer.Epoch)
s.closeAll(closeExpired)
return answer.Epoch, true
}
if answer.Epoch < cursor {
// The control plane's feed never goes back without saying reset; an answer that does is not one this node can act on.
s.log.Warn("feed_behind_cursor", "cursor", cursor, "epoch", answer.Epoch)
return cursor, false
}
events := append([]FeedEvent(nil), answer.Events...)
sort.SliceStable(events, func(i, j int) bool { return events[i].Epoch < events[j].Epoch })
next := cursor
applied := false
for _, event := range events {
if event.Epoch <= next {
continue // a duplicate, or an event this node has already applied
}
s.applyEvent(event)
next, applied = event.Epoch, true
}
return max(next, answer.Epoch), applied
}
func (s *Server) applyEvent(event FeedEvent) {
s.log.Info("feed_event", "type", event.Type, "epoch", event.Epoch)
s.met.feedEvents.inc(event.Type)
// Noted before the table is looked at: a HELLO that joins the table after that look has the note to read, and one that joined before it is found.
s.noteEvent(event)
switch event.Type {
case feedKill:
s.setKilled(true)
return
case feedKillClear:
if s.killed.Swap(false) {
s.log.Warn("kill_switch_cleared")
}
return
}
reason := map[string]string{feedRevoke: closeRevoked, feedRelease: closeReleased, feedLicenseEnd: closeRevoked}[event.Type]
hosts := s.routes.hostsOfLabel(event.Label)
for _, cc := range s.routes.controls(event.Label) {
if cc.answeredAfter(event.At) {
continue // the answer the control plane gave this connection already knew
}
switch event.Type {
case feedPause:
cc.pause()
s.routes.killStreams(cc.hostName(), closePaused, true)
s.refreshAsync(cc)
case feedResume:
s.refreshAsync(cc)
default:
cc.close(reason)
s.routes.killStreams(cc.hostName(), reason, true)
}
}
if reason != "" {
// Streams of a host whose control connection is already gone (the device dropped, the streams did not yet).
for _, host := range hosts {
s.routes.killStreams(host, reason, false)
}
}
}
// ---------------------------------------------------------------------------------------------------------------------------- events during authorize
// feedWatch collects the feed events that are applied while one exchange waits for the control plane. The answer it gets was decided before them, so
// it cannot be served as it is: a revoke applied meanwhile found no connection to close (a HELLO is not in the routing table until its OK), and a
// pause applied meanwhile to a connection that is renewing is written over by the answer's status. The notes live exactly as long as the call: nothing
// is kept for an exchange that is not waiting.
type feedWatch struct {
s *Server
mu sync.Mutex
notes []FeedEvent
}
func (s *Server) watchFeed() *feedWatch {
w := &feedWatch{s: s}
s.watchMu.Lock()
s.watches[w] = struct{}{}
s.watchMu.Unlock()
return w
}
func (w *feedWatch) stop() {
w.s.watchMu.Lock()
delete(w.s.watches, w)
w.s.watchMu.Unlock()
}
// noteEvent hands an applied event to every exchange that is waiting for authorize. The kill switch and a reset are not among them: they close every
// connection, the exchanges in flight included.
func (s *Server) noteEvent(event FeedEvent) {
switch event.Type {
case feedRevoke, feedRelease, feedLicenseEnd, feedPause, feedResume:
default:
return
}
s.watchMu.Lock()
defer s.watchMu.Unlock()
for w := range s.watches {
w.mu.Lock()
w.notes = append(w.notes, event)
w.mu.Unlock()
}
}
// feedVerdict is what the events applied during a call mean for its answer.
type feedVerdict struct {
cut string // the answer must not be served: the CLOSE reason of the event that overtook it
err string // what a device that has not been told OK yet is answered
paused bool // the last pause or resume event of the host was a pause
refresh bool // a pause or a resume came: the control plane is asked again, its answer decides
}
// verdict reads the notes that concern host, leaving out those older than the answer (by the control plane's own clock, as answeredAfter does: the
// answer already knew them).
func (w *feedWatch) verdict(host string, serverTime int64) feedVerdict {
w.mu.Lock()
defer w.mu.Unlock()
var v feedVerdict
for _, n := range w.notes {
if !hostHasLabel(host, n.Label) || serverTime > n.At {
continue
}
switch n.Type {
case feedRevoke, feedRelease:
if v.cut == "" {
v.cut, v.err = map[string]string{feedRevoke: closeRevoked, feedRelease: closeReleased}[n.Type], codeDeviceRevoked
}
case feedLicenseEnd:
if v.cut == "" {
v.cut, v.err = closeRevoked, codeLicenseInactive
}
case feedPause:
v.paused, v.refresh = true, true
case feedResume:
v.paused, v.refresh = false, true
}
}
return v
}
// answeredAfter reports whether the control plane's last answer for this connection is newer than an event stamped at (epoch seconds).
func (c *ctrl) answeredAfter(at int64) bool {
c.mu.Lock()
defer c.mu.Unlock()
return c.authAt > at
}
// setKilled turns the kill switch on: every control connection is closed (CLOSE killed) and every stream killed. While it is on no device is served.
func (s *Server) setKilled(on bool) {
if !on {
s.killed.Store(false)
return
}
if s.killed.Swap(true) {
return
}
s.log.Warn("kill_switch_on")
s.closeAll(closeKilled)
}
// closeAll closes every control connection with reason and kills every client stream: the kill switch and a feed reset.
func (s *Server) closeAll(reason string) {
s.mu.Lock()
live := make([]*ctrl, 0, len(s.conns))
for cc := range s.conns {
live = append(live, cc)
}
s.mu.Unlock()
for _, cc := range live {
cc.close(reason)
}
s.routes.killAllStreams(reason)
}
// ---------------------------------------------------------------------------------------------------------------------------- refresh of a connection
// refreshAsync asks the control plane again about one connection (after a pause or a resume event), in the background: the answer decides its status,
// and its signed statement (which carries the pause or the resume) is forwarded to the device. Events that arrive while a refresh runs are folded into
// one more. Until the control plane answers, a paused connection stays paused: the node never lifts a pause by itself.
func (s *Server) refreshAsync(cc *ctrl) {
cc.mu.Lock()
if cc.refreshing {
cc.refreshAgain = true
cc.mu.Unlock()
return
}
cc.refreshing = true
cc.mu.Unlock()
if !s.goBackground(func() { s.refreshLoop(cc) }) {
cc.mu.Lock()
cc.refreshing = false
cc.mu.Unlock()
}
}
// goBackground runs task as one of the node's background tasks, unless the node has ended: the wait for them (shutdown) must not start while one is
// being added.
func (s *Server) goBackground(task func()) bool {
s.mu.Lock()
defer s.mu.Unlock()
if s.ctx.Err() != nil {
return false
}
s.bg.Add(1)
go func() {
defer s.bg.Done()
task()
}()
return true
}
func (s *Server) refreshLoop(cc *ctrl) {
ctx, cancel := context.WithCancel(s.ctx)
defer cancel()
go func() {
select {
case <-cc.done:
cancel()
case <-ctx.Done():
}
}()
failures := 0
for ctx.Err() == nil {
cc.mu.Lock()
cc.refreshAgain = false
cc.mu.Unlock()
if err := s.refreshOnce(cc); err != nil {
failures++
if !s.clk.sleep(ctx, jitter(backoff(failures), 20)) {
break
}
continue
}
failures = 0
cc.mu.Lock()
again := cc.refreshAgain
if !again {
cc.refreshing = false
}
cc.mu.Unlock()
if !again {
return
}
}
cc.mu.Lock()
cc.refreshing = false
cc.mu.Unlock()
}
var errRefreshLater = errors.New("the control plane did not answer")
// refreshOnce is one authorize call for an established connection. It returns errRefreshLater when the control plane could not be asked (try again),
// and nil when the connection's state is settled (updated, or closed).
func (s *Server) refreshOnce(cc *ctrl) error {
cc.mu.Lock()
req := AuthorizeRequest{DeviceID: cc.sub, SessionJTI: cc.jti, Reason: "renew", Node: s.cfg.NodeID}
host := cc.host
cc.mu.Unlock()
if s.killed.Load() {
return nil
}
resp, err := s.authorize(req)
if err == nil {
if cerr := checkAuthorize(resp, req); cerr != nil {
s.health.bad(s.now())
err = &ControlError{Status: 502, Code: "bad_response"}
}
}
if err != nil {
if isRefusal(err) {
// The control plane says this device is no longer served (revoked, license over, not enrolled): it is cut.
s.log.Info("refresh_refused", "conn", cc.id, "reason", errForControl(err))
cc.close(closeRevoked)
s.routes.killStreams(host, closeRevoked, true)
return nil
}
return errRefreshLater
}
if resp.E2EKill {
s.setKilled(true)
return nil
}
if resp.Host != host {
s.log.Warn("refresh_host_changed", "conn", cc.id)
cc.close(closeRevoked)
s.routes.killStreams(host, closeRevoked, true)
return nil
}
cc.setStatus(resp)
s.routes.setGated(cc, resp.Status == statusGated)
if resp.Status == statusPaused {
s.routes.killStreams(host, closePaused, true)
}
if line := statementLine(resp.Access); resp.Access != "" && line != "" {
if err := cc.write(line); err != nil {
cc.close("write_error")
return nil
}
}
s.log.Info("control_refreshed", "conn", cc.id, "status", resp.Status)
return nil
}
// sweepLoop forgets, once a minute, what the tables of sources, hints and gated days hold that nobody needs any more, so an idle node's memory shrinks.
func (s *Server) sweepLoop(ctx context.Context) {
for s.clk.sleep(ctx, time.Minute) {
s.ips.Sweep()
s.hints.Sweep()
s.gates.Sweep()
}
}
files/tunnel/feed_test.go 962 lines · 33.5 KB
package main
import (
"context"
"errors"
"net"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
)
// The feed consumer: what the control plane says the node must stop serving, applied within the deadline, once, in order.
// scriptedFeed is a feed the tests write by hand. By default it answers like the control plane's: the events after the cursor, in order, with the head
// as the next cursor; a test overrides what it answers to feed the node duplicates, disorder and nonsense.
type scriptedFeed struct {
mu sync.Mutex
events []FeedEvent
head int64
fail error
custom func(since int64) FeedAnswer
cursor []int64 // the cursor of every poll
block chan struct{}
}
func (f *scriptedFeed) push(events ...FeedEvent) {
f.mu.Lock()
defer f.mu.Unlock()
for _, e := range events {
if e.Epoch == 0 {
f.head++
e.Epoch = f.head
}
f.head = max(f.head, e.Epoch)
f.events = append(f.events, e)
}
}
func (f *scriptedFeed) set(custom func(since int64) FeedAnswer) {
f.mu.Lock()
f.custom = custom
f.mu.Unlock()
}
func (f *scriptedFeed) polls() []int64 {
f.mu.Lock()
defer f.mu.Unlock()
return append([]int64(nil), f.cursor...)
}
func (f *scriptedFeed) Feed(ctx context.Context, since int64, wait time.Duration) (FeedAnswer, error) {
f.mu.Lock()
f.cursor = append(f.cursor, since)
fail, custom, block := f.fail, f.custom, f.block
f.mu.Unlock()
if block != nil {
select {
case <-block:
case <-ctx.Done():
return FeedAnswer{}, ctx.Err()
}
}
if fail != nil {
return FeedAnswer{}, fail
}
if custom != nil {
return custom(since), nil
}
f.mu.Lock()
defer f.mu.Unlock()
answer := FeedAnswer{Epoch: f.head, ServerTime: time.Now().Unix()}
for _, e := range f.events {
if e.Epoch > since {
answer.Events = append(answer.Events, e)
}
}
return answer, nil
}
func (e *testEnv) event(kind string, d *testDevice) FeedEvent {
ev := FeedEvent{Type: kind, At: e.now().Unix()}
if d != nil {
ev.DeviceID, ev.Label = d.id, d.label
}
return ev
}
// withFeed is a node on the real clock whose feed is f, polled every 20 ms.
func withFeed(t *testing.T, f *scriptedFeed) *testEnv {
return newTestEnv(t, func(c *Config) {
c.Feed = f
c.FeedPoll = 20 * time.Millisecond
})
}
// streamOpen opens a client stream of d and sends one request over it, leaving the connection open.
func streamOpen(t *testing.T, e *testEnv, d *testDevice) *tlsClient {
t.Helper()
c, err := e.clientTLS(d)
if err != nil {
t.Fatal(err)
}
if resp, err := postOver(c, d.host, []byte("hello")); err != nil || string(resp) != "echo:hello" {
t.Fatalf("a stream of %s: %q %v", d.host, resp, err)
}
return &tlsClient{Conn: c}
}
type tlsClient struct{ net.Conn }
// dead reports whether the node cut the stream within timeout.
func (c *tlsClient) dead(timeout time.Duration) bool {
_ = c.SetReadDeadline(time.Now().Add(timeout))
buf := make([]byte, 1)
_, err := c.Read(buf)
if err == nil {
return false
}
var ne net.Error
return !(errors.As(err, &ne) && ne.Timeout())
}
// ---------------------------------------------------------------------------------------------------------------------------- the deadline
func TestFeed_ARevokeClosesTheControlConnectionAndTheStreamsWithinTheApplyDeadline(t *testing.T) {
if policyData.Timing.FeedPollS != 15 || policyData.Timing.FeedApplyDeadlineS != 20 {
t.Fatalf("the plan says a poll every 15 s and an apply within 20 s: policy %d and %d", policyData.Timing.FeedPollS, policyData.Timing.FeedApplyDeadlineS)
}
clk := newFakeClock()
e, cp := httpNode(t, clk, nil)
d, bystander := e.newDevice(), e.newDevice()
dc, bc := e.connectOK(d), e.connectOK(bystander)
ds := dc.serve(nil)
bs := bc.serve(nil)
stream := streamOpen(t, e, d)
other := streamOpen(t, e, bystander)
defer other.Close()
cp.addEvent("revoke", d)
// Virtual time passes second by second; the node must have acted before the deadline.
elapsed := 0
for ; elapsed <= policyData.Timing.FeedApplyDeadlineS; elapsed++ {
if !e.srv.Online(d.host) {
break
}
clk.Advance(time.Second)
time.Sleep(15 * time.Millisecond)
}
if elapsed > policyData.Timing.FeedApplyDeadlineS {
t.Fatalf("the node had not applied the revoke after %d s", elapsed)
}
eventually(t, "CLOSE revoked", 2*time.Second, func() bool { return containsLine(ds.otherLines(), "CLOSE revoked") })
<-ds.done
if !stream.dead(2 * time.Second) {
t.Fatal("the stream of a revoked device is still open")
}
// nothing else was touched
if !e.srv.Online(bystander.host) || other.dead(100*time.Millisecond) || len(bs.otherLines()) != 0 {
t.Fatalf("another device was affected: lines %q", bs.otherLines())
}
// and it is not served again: the control plane would refuse its next HELLO
e.cp.set(d.id, nil)
if _, err := e.clientTLS(d); err == nil {
t.Fatal("a client reached a revoked device")
}
}
func TestFeed_ALongPollDeliversARevokeTheMomentItHappens(t *testing.T) {
e, cp := httpNode(t, nil, nil)
cp.mu.Lock()
cp.hold = 10 * time.Second
cp.mu.Unlock()
d := e.newDevice()
dc := e.connectOK(d)
ds := dc.serve(nil)
stream := streamOpen(t, e, d)
eventually(t, "a held poll", 3*time.Second, func() bool {
for _, c := range cp.feedCalls() {
if c.wait == int64(policyData.Timing.FeedPollS) {
return true
}
}
return false
})
started := time.Now()
cp.addEvent("revoke", d)
eventually(t, "CLOSE revoked", 3*time.Second, func() bool { return containsLine(ds.otherLines(), "CLOSE revoked") })
if took := time.Since(started); took > 2*time.Second {
t.Fatalf("a held poll delivered the revoke after %s", took)
}
if !stream.dead(2 * time.Second) {
t.Fatal("stream alive")
}
// The poll the node signed: the exact URL, the cursor and the hold.
for _, c := range cp.feedCalls() {
if !strings.HasPrefix(c.url, "/v1/tunnel/feed?since=") {
t.Errorf("feed URL %q", c.url)
}
}
first := cp.feedCalls()[0]
if first.since != 0 || first.wait != 0 {
t.Fatalf("the catch-up poll is the first, from the start and without a hold: %+v", first)
}
}
func TestFeed_PollsOncePerPeriodAndAdvancesItsCursor(t *testing.T) {
clk := newFakeClock()
e, cp := httpNode(t, clk, nil)
d := e.newDevice()
_ = e.connectOK(d)
// the catch-up poll, then the first long poll right after it (a control plane that does not hold it answers at once)
eventually(t, "the catch-up and the first long poll", 3*time.Second, func() bool { return len(cp.feedCalls()) == 2 })
time.Sleep(50 * time.Millisecond)
if n := len(cp.feedCalls()); n != 2 {
t.Fatalf("the node polled %d times without a period passing", n)
}
other := &testDevice{id: randomHex(16), label: label20()}
cp.addEvent("revoke", other) // nobody here: the cursor still moves
step := func(want int) {
t.Helper()
for i := 0; i < 20 && len(cp.feedCalls()) < want; i++ {
clk.Advance(time.Second)
time.Sleep(15 * time.Millisecond)
}
if n := len(cp.feedCalls()); n != want {
t.Fatalf("%d polls, want %d", n, want)
}
}
step(4) // the poll that brings the event, and the long poll that follows it at once
step(5)
calls := cp.feedCalls()
wantSince := []int64{0, 0, 0, 1, 1}
for i, c := range calls {
if c.since != wantSince[i] {
t.Errorf("poll %d asked since=%d, want %d", i, c.since, wantSince[i])
}
wantWait := int64(policyData.Timing.FeedPollS)
if i == 0 {
wantWait = 0
}
if c.wait != wantWait {
t.Errorf("poll %d asked wait=%d, want %d", i, c.wait, wantWait)
}
}
}
// ---------------------------------------------------------------------------------------------------------------------------- what each event does
func TestFeed_RevokeReleaseAndLicenseEndCloseTheConnectionAndKillTheStreams(t *testing.T) {
for kind, want := range map[string]string{feedRevoke: "CLOSE revoked", feedRelease: "CLOSE released", feedLicenseEnd: "CLOSE revoked"} {
t.Run(kind, func(t *testing.T) {
f := &scriptedFeed{}
e := withFeed(t, f)
d := e.newDevice()
dc := e.connectOK(d)
ds := dc.serve(nil)
stream := streamOpen(t, e, d)
f.push(e.event(kind, d))
eventually(t, want, 3*time.Second, func() bool { return containsLine(ds.otherLines(), want) })
if !stream.dead(2 * time.Second) {
t.Fatal("the stream is still open")
}
eventually(t, "the host unbound", 2*time.Second, func() bool { return !e.srv.Online(d.host) })
e.waitIdle()
})
}
}
func TestFeed_AStreamOfAHostWhoseConnectionIsAlreadyGoneIsStillKilled(t *testing.T) {
f := &scriptedFeed{}
e := withFeed(t, f)
d := e.newDevice()
dc := e.connectOK(d)
_ = dc.serve(nil)
stream := streamOpen(t, e, d)
_ = dc.c.Close() // the device's control connection drops, its data connection and the client's stream are still there
eventually(t, "the control connection gone", 2*time.Second, func() bool { return e.srv.routes.controls(d.label) == nil })
if stream.dead(100 * time.Millisecond) {
t.Fatal("a stream was cut because the control connection dropped: it keeps to its own limits")
}
f.push(e.event(feedRevoke, d))
if !stream.dead(3 * time.Second) {
t.Fatal("the revoke did not reach a stream that outlived its control connection")
}
}
func TestFeed_APauseStopsNewStreamsKillsTheOnesInFlightAndForwardsTheSignedStatement(t *testing.T) {
const pausedStatement = "eyJhbGciOiJFZERTQSJ9.eyJzdGF0ZSI6InBhdXNlZCJ9.c2lnbmF0dXJl"
const resumedStatement = "eyJhbGciOiJFZERTQSJ9.eyJzdGF0ZSI6InJlc3VtZWQifQ.c2lnbmF0dXJl"
f := &scriptedFeed{}
e := withFeed(t, f)
d := e.newDevice()
dc := e.connectOK(d)
ds := dc.serve(nil)
stream := streamOpen(t, e, d)
answer := func(status, access string) {
e.cp.set(d.id, &devicePlan{host: d.host, answer: func(int) AuthorizeResponse {
now := e.now()
return AuthorizeResponse{DeviceID: d.id, Label: d.label, Host: d.host, Status: status, Plan: "paid", ValidUntil: now.Unix() + 3600, ServerTime: now.Unix(),
Policy: okPolicy{20, 45, 1800, 14400, 300}, Nodes: []any{}, Access: access}
}})
}
answer(statusPaused, pausedStatement)
f.push(e.event(feedPause, d))
eventually(t, "the signed statement of the pause", 3*time.Second, func() bool { return containsLine(ds.otherLines(), "STATEMENT "+pausedStatement) })
if !stream.dead(2 * time.Second) {
t.Fatal("a stream of a paused device survived the pause")
}
if e.srv.Online(d.host) {
t.Fatal("a paused device is online")
}
if _, err := e.clientTLS(d); err == nil {
t.Fatal("a client got a stream of a paused device")
}
if !dc.try0() {
t.Fatal("the control connection of a paused device must stay (the resume arrives as a statement)")
}
if n := e.cp.callsFor(d.id, "renew"); n != 1 {
t.Fatalf("one refresh was expected, the control plane saw %d", n)
}
answer(statusActive, resumedStatement)
f.push(e.event(feedResume, d))
eventually(t, "the statement of the resume", 3*time.Second, func() bool { return containsLine(ds.otherLines(), "STATEMENT "+resumedStatement) })
eventually(t, "streams again", 2*time.Second, func() bool { return e.srv.Online(d.host) })
c, err := e.clientTLS(d)
if err != nil {
t.Fatalf("a client after the resume: %v", err)
}
_ = c.Close()
e.checkTranscript()
}
// try0 reports whether the device's connection is still up: nothing closed it.
func (dc *devConn) try0() bool {
select {
case _, ok := <-dc.feed:
return ok
case <-time.After(150 * time.Millisecond):
return true
}
}
func TestFeed_APausedConnectionStaysPausedUntilTheControlPlaneAnswers(t *testing.T) {
f := &scriptedFeed{}
e := withFeed(t, f)
d := e.newDevice()
dc := e.connectOK(d)
ds := dc.serve(nil)
// the control plane cannot be asked right now
e.cp.set(d.id, &devicePlan{host: d.host, err: &ControlError{Status: 503, Code: "end_to_end_unavailable"}})
f.push(e.event(feedPause, d))
eventually(t, "paused locally, without waiting for an answer", 3*time.Second, func() bool { return !e.srv.Online(d.host) })
f.push(e.event(feedResume, d))
time.Sleep(300 * time.Millisecond)
if e.srv.Online(d.host) {
t.Fatal("the node lifted a pause by itself, with no answer from the control plane")
}
// the answer comes: the pause is lifted
e.cp.set(d.id, &devicePlan{host: d.host, validFor: time.Hour})
eventually(t, "the refresh after the control plane is back", 6*time.Second, func() bool { return e.srv.Online(d.host) })
if e.cp.callsFor(d.id, "renew") < 2 {
t.Fatal("the node did not try again")
}
_ = ds
}
func TestFeed_AResumeThatTheControlPlaneRefusesCutsTheConnection(t *testing.T) {
f := &scriptedFeed{}
e := withFeed(t, f)
d := e.newDevice()
dc := e.connectOK(d)
ds := dc.serve(nil)
e.cp.set(d.id, &devicePlan{host: d.host, err: &ControlError{Status: 402, Code: codeLicenseInactive}})
f.push(e.event(feedResume, d))
eventually(t, "CLOSE revoked", 3*time.Second, func() bool { return containsLine(ds.otherLines(), "CLOSE revoked") })
}
func TestFeed_TheKillSwitchClosesEverythingAndNobodyIsServedUntilItIsCleared(t *testing.T) {
f := &scriptedFeed{}
e := withFeed(t, f)
a, b := e.newDevice(), e.newDevice()
ca, cb := e.connectOK(a), e.connectOK(b)
da, db := ca.serve(nil), cb.serve(nil)
sa := streamOpen(t, e, a)
sb := streamOpen(t, e, b)
f.push(FeedEvent{Type: feedKill, At: e.now().Unix()})
for _, ds := range []*devServer{da, db} {
ds := ds
eventually(t, "CLOSE killed", 3*time.Second, func() bool { return containsLine(ds.otherLines(), "CLOSE killed") })
}
if !sa.dead(2*time.Second) || !sb.dead(2*time.Second) {
t.Fatal("a stream survived the kill switch")
}
calls := len(e.cp.callLog())
c := e.connect(a)
line := c.hello(func() string { return e.session(a, nil) }, func(s, n string) string { return e.proof(a, s, n, nil) })
if line != "ERR control_unavailable" {
t.Fatalf("a HELLO while the switch is on: %q", line)
}
if len(e.cp.callLog()) != calls {
t.Fatal("the node asked the control plane about a device while the kill switch is on")
}
f.push(FeedEvent{Type: feedKillClear, At: e.now().Unix()})
eventually(t, "served again", 3*time.Second, func() bool {
cc := e.connect(a)
l := cc.hello(func() string { return e.session(a, nil) }, func(s, n string) string { return e.proof(a, s, n, nil) })
return strings.HasPrefix(l, "OK ")
})
}
func TestFeed_TheKillSwitchSeenInAnAuthorizeAnswerIsTheSameAsTheEvent(t *testing.T) {
e := newTestEnv(t, nil)
a, b := e.newDevice(), e.newDevice()
ca := e.connectOK(a)
da := ca.serve(nil)
e.cp.set(b.id, &devicePlan{host: b.host, answer: func(int) AuthorizeResponse {
now := e.now()
return AuthorizeResponse{DeviceID: b.id, Label: b.label, Host: b.host, Status: statusActive, Plan: "paid", ValidUntil: now.Unix() + 3600, ServerTime: now.Unix(),
Policy: okPolicy{20, 45, 1800, 14400, 300}, Nodes: []any{}, E2EKill: true}
}})
cb := e.connect(b)
if line := cb.hello(func() string { return e.session(b, nil) }, func(s, n string) string { return e.proof(b, s, n, nil) }); line != "ERR control_unavailable" {
t.Fatalf("a HELLO answered with the kill flag: %q", line)
}
eventually(t, "the other device told", 3*time.Second, func() bool { return containsLine(da.otherLines(), "CLOSE killed") })
}
// ---------------------------------------------------------------------------------------------------------------------------- order, duplicates, stale events
func TestFeed_DuplicatesAreAppliedOnceAndAnEventAtOrBelowTheCursorIsDropped(t *testing.T) {
f := &scriptedFeed{}
e := withFeed(t, f)
d := e.newDevice()
dc := e.connectOK(d)
ds := dc.serve(nil)
resume := e.event(feedResume, d)
resume.Epoch = 5
f.set(func(since int64) FeedAnswer {
a := FeedAnswer{Epoch: 5, ServerTime: time.Now().Unix()}
// the same event three times in one answer, and again in every answer after it: a feed that does not honor the cursor
a.Events = []FeedEvent{resume, resume, resume}
return a
})
eventually(t, "the polls", 3*time.Second, func() bool { return len(f.polls()) >= 6 })
if n := e.cp.callsFor(d.id, "renew"); n != 1 {
t.Fatalf("the same event was applied %d times", n)
}
// an older epoch than the cursor changes nothing, even when it is a revoke
old := e.event(feedRevoke, d)
old.Epoch = 3
f.set(func(since int64) FeedAnswer {
return FeedAnswer{Epoch: 5, ServerTime: time.Now().Unix(), Events: []FeedEvent{old}}
})
time.Sleep(200 * time.Millisecond)
if !e.srv.Online(d.host) || containsLine(ds.otherLines(), "CLOSE revoked") {
t.Fatal("an event below the cursor was applied")
}
// the cursor moved to the head and never back
last := int64(-1)
for _, c := range f.polls() {
if c < last {
t.Fatalf("the cursor went back: %v", f.polls())
}
last = c
}
}
func TestFeed_AnAnswerOutOfOrderIsAppliedInEpochOrder(t *testing.T) {
f := &scriptedFeed{}
e := withFeed(t, f)
d := e.newDevice()
dc := e.connectOK(d)
ds := dc.serve(nil)
// delivered as [resume 8, pause 7]: in order that is a pause and then a resume, so the device ends up served; as delivered it would end paused.
pause, resume := e.event(feedPause, d), e.event(feedResume, d)
pause.Epoch, resume.Epoch = 7, 8
var delivered atomic.Bool
f.set(func(since int64) FeedAnswer {
if since >= 8 {
return FeedAnswer{Epoch: 8, ServerTime: time.Now().Unix()}
}
delivered.Store(true)
return FeedAnswer{Epoch: 8, ServerTime: time.Now().Unix(), Events: []FeedEvent{resume, pause}}
})
eventually(t, "the answer delivered", 3*time.Second, delivered.Load)
// both events produced a refresh and, whatever the interleaving, the control plane's last answer (active) is the state
eventually(t, "refreshed", 3*time.Second, func() bool { return e.cp.callsFor(d.id, "renew") >= 1 })
eventually(t, "served", 3*time.Second, func() bool { return e.srv.Online(d.host) })
if containsLine(ds.otherLines(), "CLOSE revoked") {
t.Fatal("closed")
}
if got := f.polls(); got[len(got)-1] != 8 {
t.Fatalf("the cursor must be the highest epoch seen: %v", got)
}
}
func TestFeed_AnEventOlderThanTheLastAnswerForAConnectionIsAlreadyInThatAnswer(t *testing.T) {
f := &scriptedFeed{}
e := withFeed(t, f)
d := e.newDevice()
dc := e.connectOK(d)
ds := dc.serve(nil)
// The device's license was ended and renewed: the revoke is from before the connection's own authorize, and is replayed (a node that restarted
// replays the feed from the start). The connection must not be cut by it.
stale := e.event(feedRevoke, d)
stale.At = e.now().Unix() - 120
f.push(stale)
time.Sleep(300 * time.Millisecond)
if !e.srv.Online(d.host) || containsLine(ds.otherLines(), "CLOSE revoked") {
t.Fatal("a revoke older than the connection's last answer cut it")
}
// a revoke from now is not stale
f.push(e.event(feedRevoke, d))
eventually(t, "CLOSE revoked", 3*time.Second, func() bool { return containsLine(ds.otherLines(), "CLOSE revoked") })
}
func TestFeed_UnreadableEventsAreSkippedAndTheRestOfTheAnswerIsStillApplied(t *testing.T) {
d := &testDevice{id: randomHex(16), label: label20()}
good := map[string]any{"epoch": 4, "type": "revoke", "device_id": d.id, "label": d.label, "at": 1793000000}
doc := map[string]any{"ok": true, "v": 1, "epoch": 4, "server_time": 1793000000, "events": []any{
map[string]any{"epoch": 1, "type": "teleport", "device_id": d.id, "label": d.label, "at": 1793000000}, // a type this node does not know
map[string]any{"epoch": 2, "type": "revoke", "device_id": "not-an-id", "label": d.label, "at": 1793000000}, // a malformed device id
map[string]any{"epoch": 3, "type": "kill", "device_id": d.id, "label": nil, "at": 1793000000}, // a global event that names a device
map[string]any{"epoch": 0, "type": "revoke", "device_id": d.id, "label": d.label, "at": 1793000000}, // epoch 0
map[string]any{"epoch": 5, "type": "revoke", "device_id": d.id, "label": "UPPERCASELABELUPPERCA", "at": 1}, // a bad label
good,
}}
answer, err := parseFeedAnswer([]byte(canonicalJSON(doc)))
if err != nil {
t.Fatal(err)
}
if len(answer.Events) != 1 || answer.Events[0].Epoch != 4 || answer.Skipped != 5 {
t.Fatalf("events %+v skipped %d", answer.Events, answer.Skipped)
}
// the envelope is read strictly: a node does not act on an answer it cannot make sense of
for name, bad := range map[string]string{
"not ok": `{"ok":false,"v":1,"epoch":1,"server_time":1793000000,"events":[]}`,
"another version": `{"ok":true,"v":2,"epoch":1,"server_time":1793000000,"events":[]}`,
"no events": `{"ok":true,"v":1,"epoch":1,"server_time":1793000000}`,
"a negative head": `{"ok":true,"v":1,"epoch":-1,"server_time":1793000000,"events":[]}`,
"a repeated key": `{"ok":true,"ok":true,"v":1,"epoch":1,"server_time":1793000000,"events":[]}`,
"reset not bool": `{"ok":true,"v":1,"epoch":1,"server_time":1793000000,"events":[],"reset":"yes"}`,
} {
if _, err := parseFeedAnswer([]byte(bad)); err == nil {
t.Errorf("%s: accepted", name)
}
}
}
func TestFeed_AResetClosesEveryTunnelAndTheCursorFollowsTheAnswer(t *testing.T) {
f := &scriptedFeed{}
e := withFeed(t, f)
a, b := e.newDevice(), e.newDevice()
ca, cb := e.connectOK(a), e.connectOK(b)
da, db := ca.serve(nil), cb.serve(nil)
sa := streamOpen(t, e, a)
f.set(func(since int64) FeedAnswer {
if since < 40 {
return FeedAnswer{Epoch: 40, Reset: true, ServerTime: time.Now().Unix()}
}
return FeedAnswer{Epoch: since, ServerTime: time.Now().Unix()}
})
for _, ds := range []*devServer{da, db} {
ds := ds
eventually(t, "CLOSE after a reset", 3*time.Second, func() bool { return containsLine(ds.otherLines(), "CLOSE expired") })
}
if !sa.dead(2 * time.Second) {
t.Fatal("a stream survived a reset")
}
eventually(t, "the cursor at the answer's epoch", 3*time.Second, func() bool {
p := f.polls()
return p[len(p)-1] == 40
})
// the devices reconnect and are authorized again: nothing is held against them
if line := e.connect(a).hello(func() string { return e.session(a, nil) }, func(s, n string) string { return e.proof(a, s, n, nil) }); !strings.HasPrefix(line, "OK ") {
t.Fatalf("a device after a reset: %q", line)
}
}
func TestFeed_AnAnswerBehindTheCursorIsNotActedOn(t *testing.T) {
f := &scriptedFeed{}
e := withFeed(t, f)
d := e.newDevice()
dc := e.connectOK(d)
ds := dc.serve(nil)
f.set(func(since int64) FeedAnswer {
if since == 0 {
return FeedAnswer{Epoch: 10, ServerTime: time.Now().Unix()}
}
// the control plane (or a cache in front of it) says the head is 3 although the node has seen 10, and brings a revoke with it
ev := e.event(feedRevoke, d)
ev.Epoch = 3
return FeedAnswer{Epoch: 3, ServerTime: time.Now().Unix(), Events: []FeedEvent{ev}}
})
time.Sleep(300 * time.Millisecond)
if !e.srv.Online(d.host) || containsLine(ds.otherLines(), "CLOSE revoked") {
t.Fatal("an answer behind the cursor was acted on")
}
for _, c := range f.polls() {
if c != 0 && c != 10 {
t.Fatalf("the cursor moved to %d on a bad answer: %v", c, f.polls())
}
}
}
// ---------------------------------------------------------------------------------------------------------------------------- the feed is down, the node is starting
func TestFeed_AFeedThatIsDownDoesNotCutAnythingAndIsHeardAgainWithinOnePollPeriod(t *testing.T) {
clk := newFakeClock()
e, cp := httpNode(t, clk, nil)
d := e.newDevice()
dc := e.connectOK(d)
ds := dc.serve(nil)
stream := streamOpen(t, e, d)
cp.setDown(503)
cp.addEvent("revoke", d) // happens while the node cannot hear it
for i := 0; i < 20; i++ { // five minutes of failing polls, in 15 s steps
clk.Advance(15 * time.Second)
time.Sleep(10 * time.Millisecond)
}
if !e.srv.Online(d.host) || stream.dead(50*time.Millisecond) || containsLine(ds.otherLines(), "CLOSE revoked") {
t.Fatal("the node cut a device because its feed is unreachable: what is authenticated goes on until valid_until")
}
failed := cp.callCount(feedPath)
if failed < 5 {
t.Fatalf("the node polled %d times in five minutes of failure", failed)
}
cp.setDown(0)
elapsed := 0
for ; elapsed <= policyData.Timing.FeedApplyDeadlineS; elapsed++ {
if !e.srv.Online(d.host) {
break
}
clk.Advance(time.Second)
time.Sleep(15 * time.Millisecond)
}
if elapsed > policyData.Timing.FeedApplyDeadlineS {
t.Fatalf("the revoke was not applied within %d s of the feed coming back", elapsed)
}
}
func TestFeed_TheFirstAnswerIsAppliedBeforeTheNodeAcceptsAnyone(t *testing.T) {
f := &scriptedFeed{}
f.push(FeedEvent{Type: feedKill, At: time.Now().Unix()}) // the switch was on while the node was down
e := withFeed(t, f)
a := e.newDevice()
c := e.connect(a)
if line := c.hello(func() string { return e.session(a, nil) }, func(s, n string) string { return e.proof(a, s, n, nil) }); line != "ERR control_unavailable" {
t.Fatalf("a device accepted before the node knew the kill switch was on: %q", line)
}
if e.cp.callsFor(a.id, "") != 0 {
t.Fatal("authorize was asked while the kill switch was on")
}
}
func TestFeed_ALateFeedDoesNotKeepTheNodeFromStarting(t *testing.T) {
f := &scriptedFeed{block: make(chan struct{})}
started := time.Now()
e := newTestEnv(t, func(c *Config) {
c.Feed = f
c.FeedPoll = 20 * time.Millisecond
c.FeedStartWait = 250 * time.Millisecond
})
if took := time.Since(started); took > 3*time.Second {
t.Fatalf("the node took %s to start with a feed that does not answer", took)
}
d := e.newDevice()
_ = e.connectOK(d)
close(f.block)
}
func TestFeed_AFailingFeedIsRetriedWithinThePollPeriodNotTheBackoffCap(t *testing.T) {
// The backoff after repeated failures reaches a minute in the policy; the feed must not wait that long to hear a revoke once it is back.
if backoff(20) <= secs(policyData.Timing.FeedPollS) {
t.Fatal("this test needs a backoff longer than the poll period")
}
f := &scriptedFeed{fail: errors.New("down")}
clk := newFakeClock()
e := newEnv(t, clk, func(e *testEnv, c *Config) { c.Feed = f })
_ = e
for i := 0; i < 25; i++ {
clk.Advance(5 * time.Second)
time.Sleep(5 * time.Millisecond)
}
n := len(f.polls())
f.mu.Lock()
f.fail = nil
f.mu.Unlock()
for i := 0; i < 5; i++ {
clk.Advance(5 * time.Second)
time.Sleep(10 * time.Millisecond)
}
if len(f.polls()) <= n {
t.Fatalf("no poll within 25 s after the feed recovered (%d polls)", n)
}
}
func TestFeed_MoreThanAThousandEventsArePagedAtOnceAndTheCursorStopsAtTheLastEventGiven(t *testing.T) {
f := &scriptedFeed{}
nobody := &testDevice{id: randomHex(16), label: label20()}
for i := 0; i < 1500; i++ {
f.push(FeedEvent{Type: feedRevoke, DeviceID: nobody.id, Label: nobody.label, At: time.Now().Unix()})
}
// the control plane's answers: at most 1000 events, and `epoch` is where the cursor goes next (the last event given, not the head)
f.set(func(since int64) FeedAnswer {
a := FeedAnswer{ServerTime: time.Now().Unix()}
for e := since + 1; e <= 1500 && len(a.Events) < 1000; e++ {
a.Events = append(a.Events, FeedEvent{Epoch: e, Type: feedRevoke, DeviceID: nobody.id, Label: nobody.label, At: time.Now().Unix()})
}
a.Epoch = 1500
if len(a.Events) == 1000 && since+1000 < 1500 {
a.Epoch = since + 1000
}
return a
})
clk := newFakeClock()
_ = newEnv(t, clk, func(_ *testEnv, c *Config) { c.Feed = f })
eventually(t, "every page", 3*time.Second, func() bool { return len(f.polls()) >= 3 })
// no clock moved: the pages followed each other at once, and only the empty poll after them waits for a period
if got := f.polls(); got[0] != 0 || got[1] != 1000 || got[2] != 1500 || len(got) != 3 {
t.Fatalf("the cursor went %v, want [0 1000 1500]", got)
}
}
// ---------------------------------------------------------------------------------------------------------------------------- events during authorize
// gateAnswer makes the control plane's `from`-th answer for d (1 = the first) decide at once and arrive only when release is called: the verdict is
// from before whatever the feed says in between. Later answers are not held and carry laterStatus.
func (e *testEnv) gateAnswer(d *testDevice, from int, laterStatus string) (decided <-chan struct{}, release func()) {
dec, rel := make(chan struct{}), make(chan struct{})
var once sync.Once
release = func() { once.Do(func() { close(rel) }) }
e.t.Cleanup(release)
e.cp.set(d.id, &devicePlan{host: d.host, answer: func(n int) AuthorizeResponse {
now := e.now()
status := statusActive
if n > from {
status = laterStatus
}
resp := AuthorizeResponse{DeviceID: d.id, Label: d.label, Host: d.host, Status: status, Plan: "paid", ValidUntil: now.Unix() + 3600, ServerTime: now.Unix(),
Policy: okPolicy{20, 45, 1800, 14400, 300}, Nodes: []any{}}
if n == from {
close(dec)
<-rel
}
return resp
}})
return dec, release
}
// applied waits until the node's feed cursor has passed epoch: the event is applied.
func (f *scriptedFeed) applied(t *testing.T, epoch int64) {
t.Helper()
eventually(t, "the event applied", 3*time.Second, func() bool {
for _, c := range f.polls() {
if c >= epoch {
return true
}
}
return false
})
}
func TestFeed_ACutThatIsAppliedWhileAHelloWaitsForTheControlPlaneIsNotLost(t *testing.T) {
for kind, want := range map[string]string{feedRevoke: "ERR device_revoked", feedRelease: "ERR device_revoked", feedLicenseEnd: "ERR license_inactive"} {
t.Run(kind, func(t *testing.T) {
f := &scriptedFeed{}
e := withFeed(t, f)
d := e.newDevice()
decided, release := e.gateAnswer(d, 1, statusActive)
hello := e.helloAsync(d)
<-decided
f.push(e.event(kind, d))
f.applied(t, 1)
release()
select {
case line := <-hello:
if line != want {
t.Fatalf("the HELLO whose answer the %s overtook was answered %q, want %q", kind, line, want)
}
case <-time.After(5 * time.Second):
t.Fatal("no answer")
}
time.Sleep(100 * time.Millisecond)
if e.srv.Online(d.host) {
t.Fatal("a device cut while it was authorizing is online")
}
e.checkTranscript()
})
}
}
func TestFeed_AnEventThatDoesNotConcernAHelloWaitingForTheControlPlaneLeavesItAlone(t *testing.T) {
f := &scriptedFeed{}
e := withFeed(t, f)
d, other := e.newDevice(), e.newDevice()
decided, release := e.gateAnswer(d, 1, statusActive)
hello := e.helloAsync(d)
<-decided
f.push(e.event(feedRevoke, other)) // another device
old := e.event(feedRevoke, d) // this one, but from before the answer: the answer already knew it
old.At = e.now().Unix() - 60
f.push(old)
f.applied(t, 2)
release()
select {
case line := <-hello:
if !strings.HasPrefix(line, "OK ") {
t.Fatalf("the HELLO was answered %q", line)
}
case <-time.After(5 * time.Second):
t.Fatal("no answer")
}
if !e.srv.Online(d.host) {
t.Fatal("the device is not online")
}
e.checkTranscript()
}
func TestFeed_APauseThatIsAppliedWhileAHelloWaitsForTheControlPlaneHoldsAndIsRefreshed(t *testing.T) {
f := &scriptedFeed{}
e := withFeed(t, f)
d := e.newDevice()
decided, release := e.gateAnswer(d, 1, statusPaused)
hello := e.helloAsync(d)
<-decided
f.push(e.event(feedPause, d))
f.applied(t, 1)
release()
select {
case line := <-hello:
if !strings.HasPrefix(line, "OK ") {
t.Fatalf("the HELLO was answered %q", line)
}
case <-time.After(5 * time.Second):
t.Fatal("no answer")
}
if e.srv.Online(d.host) {
t.Fatal("a device paused while it was authorizing is served")
}
eventually(t, "the control plane asked again", 3*time.Second, func() bool { return e.cp.callsFor(d.id, "renew") >= 1 })
time.Sleep(100 * time.Millisecond)
if e.srv.Online(d.host) {
t.Fatal("the refresh, which says paused, lifted the pause")
}
}
func TestFeed_APauseAppliedWhileARenewWaitsForTheControlPlaneIsNotWrittenOverByItsAnswer(t *testing.T) {
f := &scriptedFeed{}
e := withFeed(t, f)
d := e.newDevice()
dc := e.connectOK(d)
decided, release := e.gateAnswer(d, 2, statusPaused) // the HELLO was the first call; plain plan afterwards
e.cp.set(d.id, e.cp.plans[d.id])
renewed := make(chan string, 1)
go func() {
dc.session = e.session(d, nil)
dc.send("RENEW " + dc.session)
line, err := dc.try(5 * time.Second)
if err != nil || !strings.HasPrefix(line, "CHALLENGE ") {
renewed <- "no challenge: " + line
return
}
m, _ := parseLine(line, dirN2D)
dc.send("PROOF " + e.proof(d, dc.session, m.str("nonce"), nil))
line, _ = dc.try(5 * time.Second)
renewed <- line
}()
<-decided
f.push(e.event(feedPause, d))
f.applied(t, 1)
release()
select {
case line := <-renewed:
if !strings.HasPrefix(line, "OK ") {
t.Fatalf("the RENEW was answered %q", line)
}
case <-time.After(5 * time.Second):
t.Fatal("no answer")
}
if e.srv.Online(d.host) {
t.Fatal("the RENEW's answer, decided before the pause, lifted it")
}
}
func TestFeed_TheKillSwitchThatGoesOnWhileAHelloWaitsForTheControlPlaneDoesNotServeIt(t *testing.T) {
f := &scriptedFeed{}
e := withFeed(t, f)
d := e.newDevice()
decided, release := e.gateAnswer(d, 1, statusActive)
hello := e.helloAsync(d)
<-decided
f.push(e.event(feedKill, nil))
f.applied(t, 1)
release()
select {
case line := <-hello:
if strings.HasPrefix(line, "OK ") {
t.Fatalf("the HELLO was answered %q with the kill switch on", line)
}
case <-time.After(5 * time.Second):
t.Fatal("no answer")
}
if e.srv.Online(d.host) {
t.Fatal("a device is online with the kill switch on")
}
}
func TestFeed_AResetThatIsAppliedWhileAHelloWaitsForTheControlPlaneDoesNotServeItEither(t *testing.T) {
f := &scriptedFeed{}
e := withFeed(t, f)
d := e.newDevice()
decided, release := e.gateAnswer(d, 1, statusActive)
var reset atomic.Bool
f.set(func(since int64) FeedAnswer {
if !reset.Swap(true) {
return FeedAnswer{Reset: true, Epoch: 7, ServerTime: time.Now().Unix()}
}
return FeedAnswer{Epoch: 7, ServerTime: time.Now().Unix()}
})
hello := e.helloAsync(d)
<-decided
time.Sleep(300 * time.Millisecond) // the reset is applied by the first polls
release()
select {
case line := <-hello:
if strings.HasPrefix(line, "OK ") {
t.Fatalf("a HELLO overtaken by a reset was answered %q", line)
}
case <-time.After(5 * time.Second):
t.Fatal("no answer")
}
if e.srv.Online(d.host) {
t.Fatal("a device is online after the reset overtook its authorization")
}
}
func TestFeed_ARefreshAfterShutdownStartsNothing(t *testing.T) {
f := &scriptedFeed{}
e := withFeed(t, f)
d := e.newDevice()
e.connectOK(d)
cc, _ := e.srv.routes.lookup(d.host)
e.srv.Shutdown(time.Second)
e.srv.refreshAsync(cc)
cc.mu.Lock()
running := cc.refreshing
cc.mu.Unlock()
if running {
t.Fatal("a refresh was started on a node that has ended")
}
}
files/tunnel/gated.go 112 lines · 3.4 KB
package main
import (
"sync"
"time"
)
// The gated thin pipe (the design plan, sections 3 and 5). A device whose trial ended, or whose license lapsed within the policy's cut-off, is
// authorized as gated: it is served, but only a little (policy gated.bytes_per_day and gated.streams_per_day), enough for its paywall card to reach
// ChatGPT. The device's own gate is not a boundary against its owner (he can edit his files), so THIS is: the node counts, and it counts on every
// status the control plane gives, a RENEW included, so a device that turns gated in the middle of a connection is held to the budget from that moment
// on, its streams already open included (they are cut when the bytes run out).
//
// The count is per UTC day and per device host, in memory. What a device used while gated is not forgotten by reconnecting; it is forgotten at midnight.
// dayUsage is what one device host used of its gated budget today.
type dayUsage struct {
mu sync.Mutex
day int64
bytes int64
streams int
users int // streams holding this record: it is never swept while there are any
}
// rollLocked starts a new day's count when the day changed.
func (u *dayUsage) rollLocked(now time.Time) {
if day := now.Unix() / 86400; day != u.day {
u.day, u.bytes, u.streams = day, 0, 0
}
}
// admit counts a new stream of a gated device, or refuses it (reason says which budget is spent). A byte budget that is already spent also refuses:
// the stream would get nothing.
func (u *dayUsage) admit(now time.Time, bytesMax int64, streamsMax int) (ok bool, reason string) {
u.mu.Lock()
defer u.mu.Unlock()
u.rollLocked(now)
switch {
case u.streams >= streamsMax:
return false, "gated_streams"
case u.bytes >= bytesMax:
return false, "gated_bytes"
}
u.streams++
return true, ""
}
// charge takes n bytes of today's budget and returns how many of them may be forwarded: all of them while there is budget, fewer at the edge, none
// after it.
func (u *dayUsage) charge(now time.Time, n, bytesMax int64) int64 {
u.mu.Lock()
defer u.mu.Unlock()
u.rollLocked(now)
take := min(n, max(bytesMax-u.bytes, 0))
u.bytes += take
return take
}
// gateTable keeps the usage of every device host that was gated today.
type gateTable struct {
clk clock
mu sync.Mutex
byHost map[string]*dayUsage
}
func newGateTable(clk clock) *gateTable { return &gateTable{clk: clk, byHost: map[string]*dayUsage{}} }
// acquire gives the usage record of host (made when there is none) and counts the caller as a user; release gives it back.
func (g *gateTable) acquire(host string) *dayUsage {
g.mu.Lock()
defer g.mu.Unlock()
u := g.byHost[host]
if u == nil {
u = &dayUsage{day: g.clk.Now().Unix() / 86400}
g.byHost[host] = u
}
u.mu.Lock()
u.users++
u.mu.Unlock()
return u
}
func (g *gateTable) release(u *dayUsage) {
u.mu.Lock()
u.users--
u.mu.Unlock()
}
// Sweep forgets the records nobody holds that have nothing to remember: a previous day's, or an empty one (a background task calls it).
func (g *gateTable) Sweep() {
now := g.clk.Now()
g.mu.Lock()
defer g.mu.Unlock()
for host, u := range g.byHost {
u.mu.Lock()
if u.users == 0 {
u.rollLocked(now)
if u.bytes == 0 && u.streams == 0 {
delete(g.byHost, host)
}
}
u.mu.Unlock()
}
}
// size is the number of device hosts with a record, for the metrics.
func (g *gateTable) size() int {
g.mu.Lock()
defer g.mu.Unlock()
return len(g.byHost)
}
files/tunnel/gated_test.go 269 lines · 8.8 KB
package main
import (
"strings"
"testing"
"time"
)
// The gated thin pipe (gated.go): a device that is gated is served only a little, counted by the node on every status the control plane gives.
func TestGated_DayUsageCountsAdmitsAndRollsAtMidnight(t *testing.T) {
now := time.Date(2026, 10, 26, 23, 59, 0, 0, time.UTC)
u := &dayUsage{day: now.Unix() / 86400}
for i := 0; i < 3; i++ {
if ok, why := u.admit(now, 1000, 3); !ok {
t.Fatalf("stream %d: %s", i, why)
}
}
if ok, why := u.admit(now, 1000, 3); ok || why != "gated_streams" {
t.Fatalf("the fourth stream: %v %q", ok, why)
}
if got := u.charge(now, 600, 1000); got != 600 {
t.Fatalf("%d", got)
}
if got := u.charge(now, 600, 1000); got != 400 {
t.Fatalf("the edge gives %d, want the 400 that are left", got)
}
if got := u.charge(now, 1, 1000); got != 0 {
t.Fatalf("after the edge %d", got)
}
u2 := &dayUsage{day: now.Unix() / 86400, bytes: 1000}
if ok, why := u2.admit(now, 1000, 3); ok || why != "gated_bytes" {
t.Fatalf("a stream with no byte budget left: %v %q", ok, why)
}
next := now.Add(2 * time.Minute) // past midnight UTC
if ok, _ := u.admit(next, 1000, 3); !ok {
t.Fatal("the new day did not reset the stream count")
}
if got := u.charge(next, 700, 1000); got != 700 {
t.Fatalf("the new day did not reset the bytes: %d", got)
}
}
func TestGated_SweepKeepsWhatIsHeldOrUsedToday(t *testing.T) {
fc := newFakeClock()
g := newGateTable(newClock(fc.Now, fc.AfterFunc))
held := g.acquire("held.example")
used := g.acquire("used.example")
used.charge(fc.Now(), 10, 1000)
g.release(used)
idle := g.acquire("idle.example")
g.release(idle)
g.Sweep()
if g.size() != 2 {
t.Fatalf("%d records after the sweep, want the held one and the used one", g.size())
}
fc.Advance(25 * time.Hour)
g.Sweep()
if g.size() != 1 {
t.Fatalf("%d records a day later, want only the held one", g.size())
}
g.release(held)
g.Sweep()
if g.size() != 0 {
t.Fatalf("%d", g.size())
}
}
func gatedEnv(t *testing.T, budget int64, streams int) *testEnv {
return newTestEnv(t, func(c *Config) { c.GatedBytesPerDay, c.GatedStreamsPerDay = budget, streams })
}
func (e *testEnv) gatedBytes(d *testDevice) int64 {
e.srv.gates.mu.Lock()
u := e.srv.gates.byHost[d.host]
e.srv.gates.mu.Unlock()
if u == nil {
return 0
}
u.mu.Lock()
defer u.mu.Unlock()
return u.bytes
}
func TestGated_ADeviceStopsAtItsByteBudgetAndTheStreamIsCutThere(t *testing.T) {
const budget = 60_000
e := gatedEnv(t, budget, 200)
d := e.newDevice()
e.plan(d).status, e.plan(d).plan, e.plan(d).validFor = statusGated, "trial", 100*time.Hour // (the test moves the node's clock a day)
dc := e.connectOK(d)
if body, _ := dc.ok["body"].(map[string]any); body["status"] != "gated" {
t.Fatalf("the OK says %v", dc.ok)
}
dc.serve(nil)
c, err := e.clientTLS(d)
if err != nil {
t.Fatal(err)
}
defer c.Close()
if resp, err := postOver(c, d.host, []byte("a thin pipe is enough for a card")); err != nil || string(resp) != "echo:a thin pipe is enough for a card" {
t.Fatalf("a small exchange of a gated device: %q %v", resp, err)
}
if _, err := postOver(c, d.host, make([]byte, 400_000)); err == nil {
t.Fatal("a body far past the budget went through")
}
e.waitIdle()
if got := e.gatedBytes(d); got != budget {
t.Fatalf("the node counted %d bytes, want exactly the budget %d", got, budget)
}
up, down := e.metricValue(`all_tunnel_bytes_total{direction="up"}`), e.metricValue(`all_tunnel_bytes_total{direction="down"}`)
if up+down != budget {
t.Fatalf("%d bytes crossed the node (up %d, down %d), want exactly %d", up+down, up, down, budget)
}
if e.metricValue(`all_tunnel_gated_cut_total`) != 1 || e.metricValue(`all_tunnel_stream_closed_total{reason="gated_bytes"}`) != 1 {
t.Fatalf("metrics:\n%s", e.srv.renderMetrics())
}
if e.logs.Count(`"reason":"gated_bytes"`) < 1 {
t.Fatalf("logs:\n%s", e.logs.String())
}
// a new stream the same day gets nothing: the byte budget is spent
if _, err := e.clientTLS(d); err == nil {
t.Fatal("a stream was admitted with the day's bytes spent")
}
if e.rejected("gated_bytes") != 1 {
t.Fatalf("counted %d", e.rejected("gated_bytes"))
}
// reconnecting does not buy a new budget
dc.c.Close()
dc2 := e.connectOK(d)
dc2.serve(nil)
if _, err := e.clientTLS(d); err == nil {
t.Fatal("a reconnected device got a fresh budget")
}
// midnight does
e.offsetNS.Add(int64(24 * time.Hour))
c3, err := e.clientTLS(d)
if err != nil {
t.Fatalf("the next day: %v", err)
}
defer c3.Close()
if resp, err := postOver(c3, d.host, []byte("tomorrow")); err != nil || string(resp) != "echo:tomorrow" {
t.Fatalf("%q %v", resp, err)
}
e.checkTranscript()
}
func TestGated_AtMostTheDaysNumberOfStreams(t *testing.T) {
e := gatedEnv(t, 50<<20, 3)
d := e.newDevice()
e.plan(d).status, e.plan(d).plan = statusGated, "trial"
dc := e.connectOK(d)
dc.serve(nil)
for i := 0; i < 3; i++ {
c, err := e.clientTLS(d)
if err != nil {
t.Fatalf("stream %d: %v", i, err)
}
if _, err := postOver(c, d.host, []byte("x")); err != nil {
t.Fatal(err)
}
c.Close()
}
if _, err := e.clientTLS(d); err == nil || !strings.Contains(err.Error(), "internal error") {
t.Fatalf("the fourth stream of the day: %v", err)
}
if e.rejected("gated_streams") != 1 {
t.Fatalf("counted %d", e.rejected("gated_streams"))
}
// a device that is not gated is not counted at all
other := e.newDevice()
oc := e.connectOK(other)
oc.serve(nil)
for i := 0; i < 6; i++ {
c, err := e.clientTLS(other)
if err != nil {
t.Fatalf("an active device, stream %d: %v", i, err)
}
c.Close()
}
}
func TestGated_ADeviceThatBecomesGatedAtARenewIsHeldToTheBudgetOnTheStreamsAlreadyOpen(t *testing.T) {
const budget = 80_000
e := gatedEnv(t, budget, 200)
d := e.newDevice()
dc := e.connectOK(d) // active, paid: the trial is still on
dc.serve(nil)
c, err := e.clientTLS(d)
if err != nil {
t.Fatal(err)
}
defer c.Close()
// far more than the budget goes through while the device is active
for i := 0; i < 3; i++ {
if resp, err := postOver(c, d.host, make([]byte, 100_000)); err != nil || len(resp) != 100_005 {
t.Fatalf("an active device's exchange %d: %d bytes, %v", i, len(resp), err)
}
}
if e.gatedBytes(d) != 0 {
t.Fatalf("an active device was counted: %d", e.gatedBytes(d))
}
// the trial ends: the control plane says gated at the next RENEW, and the node holds the open stream to the budget from there on
e.plan(d).status, e.plan(d).plan = statusGated, "trial"
dc.drainFeed()
if r := dc.renew(); !strings.HasPrefix(r, "OK ") || !strings.Contains(r, `"status":"gated"`) {
t.Fatalf("renew: %q", r)
}
if resp, err := postOver(c, d.host, []byte("a card")); err != nil || string(resp) != "echo:a card" {
t.Fatalf("a small exchange after the RENEW: %q %v", resp, err)
}
if _, err := postOver(c, d.host, make([]byte, 300_000)); err == nil {
t.Fatal("the stream that was already open went past the budget after the device became gated")
}
e.waitIdle()
if got := e.gatedBytes(d); got != budget {
t.Fatalf("counted %d, want exactly %d", got, budget)
}
// and back to active at the next RENEW: nothing is counted or cut any more, what was used stays on the day's record
e.plan(d).status, e.plan(d).plan = statusActive, "paid"
if r := dc.renew(); !strings.HasPrefix(r, "OK ") || !strings.Contains(r, `"status":"active"`) {
t.Fatalf("renew: %q", r)
}
c2, err := e.clientTLS(d)
if err != nil {
t.Fatal(err)
}
defer c2.Close()
if resp, err := postOver(c2, d.host, make([]byte, 200_000)); err != nil || len(resp) != 200_005 {
t.Fatalf("a device that is active again: %d bytes, %v", len(resp), err)
}
if e.gatedBytes(d) != budget {
t.Fatalf("an active device was counted again: %d", e.gatedBytes(d))
}
e.checkTranscript()
}
func TestGated_AFeedRefreshThatMakesTheHostGatedAlsoHoldsItsStreams(t *testing.T) {
const budget = 70_000
e, cp := httpNode(t, nil, func(c *Config) { c.GatedBytesPerDay = budget })
cp.mu.Lock()
cp.hold = 10 * time.Second
cp.mu.Unlock()
d := e.newDevice()
dc := e.connectOK(d)
dc.serve(nil)
c, err := e.clientTLS(d)
if err != nil {
t.Fatal(err)
}
defer c.Close()
if _, err := postOver(c, d.host, make([]byte, 100_000)); err != nil {
t.Fatalf("while active: %v", err)
}
// a resume event after the control plane turned the device gated (the node asks authorize again, outside any RENEW of the device)
e.plan(d).status, e.plan(d).plan = statusGated, "trial"
cp.addEvent("resume", d)
eventually(t, "the refresh asked authorize again", 5*time.Second, func() bool { return e.cp.callsFor(d.id, "renew") >= 1 })
eventually(t, "the host is gated", 5*time.Second, func() bool {
cc, he := e.srv.routes.lookup(d.host)
return cc != nil && he.gated.Load()
})
if _, err := postOver(c, d.host, make([]byte, 300_000)); err == nil {
t.Fatal("the stream was not held to the budget after the refresh")
}
e.waitIdle()
if got := e.gatedBytes(d); got != budget {
t.Fatalf("counted %d, want %d", got, budget)
}
}
files/tunnel/go.mod 3 lines · 31 B
module all-e2e/tunnel
go 1.24
files/tunnel/hardening_helpers_test.go 151 lines · 5.1 KB
package main
import (
"crypto/tls"
"errors"
"fmt"
"io"
"math/rand/v2"
"net"
"net/http"
"net/netip"
"strings"
"sync"
"testing"
"time"
)
// Every test client of this package comes from loopback, which is ONE source. The hardening tests need many: virtualIPs gives each connection the source
// address the test names. The test dials from a local port it chose and records which address that port stands for; the node's peerOf hook reads the
// record by the connection's remote port. (Nothing of this is in the production build: Config.peerOf is nil there.)
type virtualIPs struct {
mu sync.Mutex
byPort map[int]netip.Addr
next int
}
func newVirtualIPs() *virtualIPs {
return &virtualIPs{byPort: map[int]netip.Addr{}, next: 20000 + rand.IntN(6000)}
}
// peerOf is the node's hook: the address a port stands for, or the real remote address of a connection nobody chose one for.
func (v *virtualIPs) peerOf(c net.Conn) (netip.Addr, bool) {
if ap, err := netip.ParseAddrPort(c.RemoteAddr().String()); err == nil {
v.mu.Lock()
addr, ok := v.byPort[int(ap.Port())]
v.mu.Unlock()
if ok {
return addr, true
}
}
return peerAddr(c)
}
// dial opens a TCP connection to addr that the node sees as coming from ip.
func (v *virtualIPs) dial(ip, addr string) (net.Conn, error) {
want := netip.MustParseAddr(ip)
var last error
for try := 0; try < 400; try++ {
v.mu.Lock()
port := v.next
if v.next++; v.next >= 32000 {
v.next = 20000
}
v.byPort[port] = want
v.mu.Unlock()
d := net.Dialer{LocalAddr: &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: port}, Timeout: 3 * time.Second}
c, err := d.Dial("tcp", addr)
if err == nil {
return c, nil
}
if !strings.Contains(err.Error(), "address already in use") && !strings.Contains(err.Error(), "can't assign requested address") {
return nil, err
}
last = err
}
return nil, last
}
// newVIPEnv is newTestEnv with source addresses a test can choose (e.vip.dial, e.clientFrom). The per-source limits are the policy's here: a test that
// wants other numbers sets them in mutate.
func newVIPEnv(t *testing.T, mutate func(*testEnv, *Config)) *testEnv {
t.Helper()
vip := newVirtualIPs()
return newEnv(t, nil, func(e *testEnv, cfg *Config) {
e.vip = vip
cfg.peerOf = vip.peerOf
cfg.PerIPConcurrent, cfg.PerIPNewPerS = DefaultConfig().PerIPConcurrent, DefaultConfig().PerIPNewPerS
cfg.UnknownSNIPerIP, cfg.LabelNewPerS, cfg.LabelNewBurst = DefaultConfig().UnknownSNIPerIP, DefaultConfig().LabelNewPerS, DefaultConfig().LabelNewBurst
cfg.LabelAnonSlice = DefaultConfig().LabelAnonSlice
if mutate != nil {
mutate(e, cfg)
}
})
}
// clientFrom connects to device d the way a ChatGPT client at address ip does and completes the TLS handshake with the device. The error is what the
// client saw: a node that drops a source without an answer shows as a closed or reset connection, one that refuses with an alert as "internal error".
func (e *testEnv) clientFrom(ip string, d *testDevice) (*tls.Conn, error) {
raw, err := e.vip.dial(ip, e.httpsAddr)
if err != nil {
return nil, err
}
tc := tls.Client(raw, &tls.Config{ServerName: d.host, RootCAs: d.pool, NextProtos: []string{"http/1.1"}, MinVersion: tls.VersionTLS12})
_ = tc.SetDeadline(time.Now().Add(4 * time.Second))
if err := tc.Handshake(); err != nil {
_ = raw.Close()
return nil, err
}
_ = tc.SetDeadline(time.Time{})
e.t.Cleanup(func() { _ = tc.Close() })
return tc, nil
}
// helloFrom connects from ip and sends a ClientHello for host (no device behind it needed); it returns the connection and what the node did, as the
// bytes it answered with until it closed (a TLS alert is 7 bytes, a dropped source is none) and whether it closed within the wait.
func (e *testEnv) helloFrom(ip, host string, wait time.Duration) (answer []byte, closed bool) {
e.t.Helper()
raw, err := e.vip.dial(ip, e.httpsAddr)
if err != nil {
e.t.Fatal(err)
}
defer raw.Close()
_, _ = raw.Write(helloOf(host, 0, 16000))
_ = raw.SetReadDeadline(time.Now().Add(wait))
b, err := io.ReadAll(raw)
var ne net.Error
return b, !(errors.As(err, &ne) && ne.Timeout())
}
// metricValue reads one series of the node's metrics text ("name" or `name{label="word"}`).
func (e *testEnv) metricValue(series string) int64 {
e.t.Helper()
for _, line := range strings.Split(e.srv.renderMetrics(), "\n") {
if rest, ok := strings.CutPrefix(line, series+" "); ok {
var n int64
if _, err := fmt.Sscan(rest, &n); err != nil {
e.t.Fatalf("series %s: %q", series, line)
}
return n
}
}
e.t.Fatalf("no series %s in the metrics:\n%s", series, e.srv.renderMetrics())
return 0
}
// rejected is how many connections the node counted as refused for reason.
func (e *testEnv) rejected(reason string) int64 {
return e.metricValue(fmt.Sprintf(`all_tunnel_conn_rejected_total{reason=%q}`, reason))
}
// httpGet reads a loopback URL of the node's metrics listener.
func httpGet(t *testing.T, url string) (int, string, http.Header) {
t.Helper()
resp, err := (&http.Client{Timeout: 5 * time.Second}).Get(url)
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
body, _ := io.ReadAll(resp.Body)
return resp.StatusCode, string(body), resp.Header
}
files/tunnel/helpers_test.go 843 lines · 25.3 KB
package main
import (
"bufio"
"bytes"
"context"
"crypto/ecdsa"
"crypto/ed25519"
"crypto/elliptic"
"crypto/rand"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"encoding/hex"
"encoding/pem"
"fmt"
"io"
"log/slog"
"math/big"
"net"
"net/http"
"os"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
)
// marker is a recognizable plaintext. If it ever shows up in a buffer the node touched while splicing a TLS stream, the node could read content.
const marker = "MARKER-9f3a1c-PLAINTEXT-MUST-NOT-LEAK-7b2e"
const (
testZone = "devices.all-e2e.test"
testNodeHost = "t1.nodes.all-e2e.test"
testNodeID = "t1"
testKID = "cp-test-1"
)
type syncBuf struct {
mu sync.Mutex
b bytes.Buffer
}
func (s *syncBuf) Write(p []byte) (int, error) {
s.mu.Lock()
defer s.mu.Unlock()
return s.b.Write(p)
}
func (s *syncBuf) String() string {
s.mu.Lock()
defer s.mu.Unlock()
return s.b.String()
}
func (s *syncBuf) Count(sub string) int { return strings.Count(s.String(), sub) }
// makeCA returns a CA pool and a leaf certificate for names, signed by it: this is what a DEVICE owns (the node never has the key), and what the
// node itself holds for its own name.
func makeCA(t testing.TB, names ...string) (*x509.CertPool, tls.Certificate) {
t.Helper()
caKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
t.Fatal(err)
}
caTmpl := &x509.Certificate{
SerialNumber: big.NewInt(1),
Subject: pkix.Name{CommonName: "test CA"},
NotBefore: time.Now().Add(-time.Hour),
NotAfter: time.Now().Add(24 * time.Hour),
IsCA: true,
KeyUsage: x509.KeyUsageCertSign,
BasicConstraintsValid: true,
}
caDER, err := x509.CreateCertificate(rand.Reader, caTmpl, caTmpl, &caKey.PublicKey, caKey)
if err != nil {
t.Fatal(err)
}
caCert, _ := x509.ParseCertificate(caDER)
leafKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
t.Fatal(err)
}
leafTmpl := &x509.Certificate{
SerialNumber: big.NewInt(2),
Subject: pkix.Name{CommonName: names[0]},
NotBefore: time.Now().Add(-time.Hour),
NotAfter: time.Now().Add(24 * time.Hour),
KeyUsage: x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
DNSNames: names,
}
leafDER, err := x509.CreateCertificate(rand.Reader, leafTmpl, caCert, &leafKey.PublicKey, caKey)
if err != nil {
t.Fatal(err)
}
pool := x509.NewCertPool()
pool.AddCert(caCert)
return pool, tls.Certificate{Certificate: [][]byte{leafDER}, PrivateKey: leafKey}
}
// ---------------------------------------------------------------------------------------------------------------------------- control plane fake
// devicePlan is what the fake control plane answers for one device.
type devicePlan struct {
host string
status string // active (default), gated or paused
plan string // paid (default) or trial
validFor time.Duration
err error // refuse instead of answering
delay time.Duration // answer late
answer func(n int) AuthorizeResponse // overrides everything, n is the call number of this device (1 = first)
}
type fakeControl struct {
env *testEnv
mu sync.Mutex
plans map[string]*devicePlan
calls []AuthorizeRequest
count map[string]int
inflight, maxInflight int // calls being answered now, and the most there ever were at once
}
func (f *fakeControl) set(deviceID string, p *devicePlan) {
f.mu.Lock()
defer f.mu.Unlock()
f.plans[deviceID] = p
}
func (f *fakeControl) callLog() []AuthorizeRequest {
f.mu.Lock()
defer f.mu.Unlock()
return append([]AuthorizeRequest(nil), f.calls...)
}
func (f *fakeControl) callsFor(deviceID, reason string) int {
n := 0
for _, c := range f.callLog() {
if c.DeviceID == deviceID && (reason == "" || c.Reason == reason) {
n++
}
}
return n
}
func (f *fakeControl) Authorize(ctx context.Context, req AuthorizeRequest) (AuthorizeResponse, error) {
f.mu.Lock()
f.calls = append(f.calls, req)
f.count[req.DeviceID]++
n := f.count[req.DeviceID]
p := f.plans[req.DeviceID]
f.inflight++
f.maxInflight = max(f.maxInflight, f.inflight)
f.mu.Unlock()
defer func() {
f.mu.Lock()
f.inflight--
f.mu.Unlock()
}()
if p == nil {
return AuthorizeResponse{}, &ControlError{Status: 401, Code: codeDeviceRevoked}
}
if p.delay > 0 {
select {
case <-time.After(p.delay):
case <-ctx.Done():
return AuthorizeResponse{}, ctx.Err()
}
}
if p.err != nil {
return AuthorizeResponse{}, p.err
}
if p.answer != nil {
return p.answer(n), nil
}
now := f.env.now()
status, plan := p.status, p.plan
if status == "" {
status = statusActive
}
if plan == "" {
plan = "paid"
}
label, _, _ := strings.Cut(p.host, ".")
t := policyData.Timing
return AuthorizeResponse{
DeviceID: req.DeviceID, Label: label, Host: p.host, Status: status, Plan: plan,
ValidUntil: now.Unix() + int64(p.validFor/time.Second), ServerTime: now.Unix(),
Policy: okPolicy{int64(t.PingIntervalS), int64(t.DeadPeerS), int64(t.StreamIdleS), int64(t.StreamMaxAgeS), int64(t.RenewBeforeValidUntilS)},
Nodes: []any{},
}, nil
}
// ---------------------------------------------------------------------------------------------------------------------------- the environment
// testDevice is a device the tests play: an id, the Ed25519 key its sessions are bound to, and (for the clients that reach it) the TLS certificate
// it holds for its host.
type testDevice struct {
id, label, host string
key ed25519.PrivateKey
pool *x509.CertPool
cert tls.Certificate
}
type testEnv struct {
t *testing.T
srv *Server
logs *syncBuf
cp *fakeControl
keys *SwapKeys
cpKey ed25519.PrivateKey
nodePool *x509.CertPool
httpsAddr, httpAddr string
vip *virtualIPs // when set, connections made with vip.dial come from the addresses the test chose (hardening tests)
offsetNS atomic.Int64 // the node's clock is the real one plus this
clk *fakeClock // when set, the node's clock and timers are this one and the tests move it by hand
tapMu sync.Mutex
tapAll bytes.Buffer
tapByID map[string]*bytes.Buffer
traceMu sync.Mutex
events []any
seq int
}
func (e *testEnv) now() time.Time {
if e.clk != nil {
return e.clk.Now()
}
return time.Now().Add(time.Duration(e.offsetNS.Load()))
}
// newTestEnv starts a node (loopback, ephemeral ports) with a fake control plane and a key set whose private half the tests hold.
func newTestEnv(t *testing.T, mutate func(*Config)) *testEnv {
return newEnv(t, nil, func(_ *testEnv, cfg *Config) {
if mutate != nil {
mutate(cfg)
}
})
}
// newClockEnv is newTestEnv on a clock the test moves by hand: every lifetime the node decides (valid_until, the grace, the feed's pace, the quiet
// period of a drain) follows it, so hours pass in an instant.
func newClockEnv(t *testing.T, mutate func(*Config)) *testEnv {
return newEnv(t, newFakeClock(), func(_ *testEnv, cfg *Config) {
if mutate != nil {
mutate(cfg)
}
})
}
// newEnv builds the node; mutate gets the environment (its fake control plane, its keys, its clock) and the configuration before the node starts.
func newEnv(t *testing.T, clk *fakeClock, mutate func(*testEnv, *Config)) *testEnv {
t.Helper()
e := &testEnv{t: t, clk: clk, logs: &syncBuf{}, tapByID: map[string]*bytes.Buffer{}}
pub, priv, err := ed25519.GenerateKey(rand.Reader)
if err != nil {
t.Fatal(err)
}
e.cpKey = priv
e.keys = NewSwapKeys(StaticKeys{testKID: pub})
e.cp = &fakeControl{env: e, plans: map[string]*devicePlan{}, count: map[string]int{}}
pool, nodeCert := makeCA(t, testNodeHost)
e.nodePool = pool
cfg := DefaultConfig()
cfg.HTTPSAddr, cfg.HTTPAddr = "127.0.0.1:0", "127.0.0.1:0"
cfg.NodeID, cfg.NodeHost = testNodeID, testNodeHost
cfg.CertSource = func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return &nodeCert, nil }
cfg.Keys, cfg.Control = e.keys, e.cp
cfg.Now = e.now
if clk != nil {
cfg.AfterFunc = clk.AfterFunc
}
cfg.MinRenewGap = 0 // tests renew in quick succession; the test of the gap sets its own
// Every test client and device comes from loopback: one source. The per-source limits (limits_test.go gives its own) would make a crowd of devices
// in a test look like one attacker, so they are out of the way here.
cfg.PerIPConcurrent, cfg.PerIPNewPerS = 1<<20, 1<<20
cfg.UnknownSNIPerIP, cfg.LabelNewPerS, cfg.LabelNewBurst = 1<<20, 1<<20, 1<<20
cfg.LabelAnonSlice = cfg.MaxStreamsPerHost // and every test client is anonymous: the slices have their own tests
cfg.HelloTimeout = 2 * time.Second
cfg.AttachTimeout = 3 * time.Second
cfg.FirstLineTimeout = 3 * time.Second
cfg.StreamIdle = 20 * time.Second
cfg.StreamMaxAge = time.Minute
cfg.HalfCloseTimeout = 5 * time.Second
cfg.ACMETimeout = 5 * time.Second
cfg.Logger = slog.New(slog.NewJSONHandler(e.logs, &slog.HandlerOptions{Level: slog.LevelDebug}))
cfg.tap = func(id, dir string, p []byte) {
e.tapMu.Lock()
defer e.tapMu.Unlock()
e.tapAll.Write(p)
b := e.tapByID[id]
if b == nil {
b = &bytes.Buffer{}
e.tapByID[id] = b
}
b.Write(p)
}
cfg.trace = func(ev map[string]any) {
// the checker reads JSON values: numbers are float64, whatever the node used
parsed, err := parseJSON(canonicalJSON(ev))
if err != nil {
panic(err)
}
e.traceMu.Lock()
e.events = append(e.events, parsed)
e.traceMu.Unlock()
}
if mutate != nil {
mutate(e, &cfg)
}
srv, err := NewServer(cfg)
if err != nil {
t.Fatal(err)
}
if err := srv.Start(); err != nil {
t.Fatal(err)
}
e.srv = srv
e.httpsAddr, e.httpAddr = srv.HTTPSAddr(), srv.HTTPAddr()
t.Cleanup(func() { srv.Shutdown(2 * time.Second) })
return e
}
func (e *testEnv) tapped(id string) []byte {
e.tapMu.Lock()
defer e.tapMu.Unlock()
if b := e.tapByID[id]; b != nil {
return append([]byte(nil), b.Bytes()...)
}
return nil
}
func (e *testEnv) tappedAll() []byte {
e.tapMu.Lock()
defer e.tapMu.Unlock()
return append([]byte(nil), e.tapAll.Bytes()...)
}
func randomHex(n int) string {
b := make([]byte, n)
_, _ = rand.Read(b)
return hex.EncodeToString(b)
}
// label20 is a label of the contracts' alphabet.
func label20() string {
const alphabet = "abcdefghijklmnopqrstuvwxyz234567"
b := make([]byte, 20)
_, _ = rand.Read(b)
for i := range b {
b[i] = alphabet[int(b[i])%len(alphabet)]
}
return string(b)
}
// newDevice makes a device and tells the fake control plane about it (host under the test zone, paid, valid for an hour unless changed).
func (e *testEnv) newDevice() *testDevice {
d := e.newDeviceNoPlan()
e.cp.set(d.id, &devicePlan{host: d.host, validFor: time.Hour})
return d
}
// newDeviceNoPlan makes a device the control plane has not been told about.
func (e *testEnv) newDeviceNoPlan() *testDevice {
e.t.Helper()
_, priv, err := ed25519.GenerateKey(rand.Reader)
if err != nil {
e.t.Fatal(err)
}
d := &testDevice{id: randomHex(16), label: label20(), key: priv}
d.host = d.label + "." + testZone
d.pool, d.cert = makeCA(e.t, d.host)
return d
}
func (e *testEnv) plan(d *testDevice) *devicePlan {
e.cp.mu.Lock()
defer e.cp.mu.Unlock()
return e.cp.plans[d.id]
}
// ---------------------------------------------------------------------------------------------------------------------------- tokens
func signJWS(key ed25519.PrivateKey, header, claims map[string]any) string {
h := b64Encode([]byte(canonicalJSON(header)))
c := b64Encode([]byte(canonicalJSON(claims)))
input := h + "." + c
return input + "." + b64Encode(ed25519.Sign(key, []byte(input)))
}
// session mints a device session for d as the control plane would (aud all-tunnel, 300 s). mutate changes the claims or the header before signing.
func (e *testEnv) session(d *testDevice, mutate func(header, claims map[string]any)) string {
return e.sessionSignedBy(e.cpKey, d, mutate)
}
// sessionSignedBy is session signed with another key (a forger's).
func (e *testEnv) sessionSignedBy(signer ed25519.PrivateKey, d *testDevice, mutate func(header, claims map[string]any)) string {
iat := e.now().Unix()
pub := d.key.Public().(ed25519.PublicKey)
header := map[string]any{"alg": "EdDSA", "typ": "device-session", "kid": testKID}
claims := map[string]any{
"typ": "device-session", "ver": 2, "iss": policyData.Identity.Issuer, "aud": policyData.Identity.AudTunnel, "sub": d.id,
"iat": iat, "exp": iat + int64(policyData.Authorization.SessionTTLS), "jti": randomHex(12), "generation": 1,
"cnf": map[string]any{"jwk": map[string]any{"kty": "OKP", "crv": "Ed25519", "x": b64Encode(pub)}},
}
if mutate != nil {
mutate(header, claims)
}
return signJWS(signer, header, claims)
}
// proof mints the DPoP proof of d over session and nonce for the node's URL.
func (e *testEnv) proof(d *testDevice, session, nonce string, mutate func(header, claims map[string]any)) string {
pub := d.key.Public().(ed25519.PublicKey)
header := map[string]any{"alg": "EdDSA", "typ": "dpop+jwt", "jwk": map[string]any{"kty": "OKP", "crv": "Ed25519", "x": b64Encode(pub)}}
claims := map[string]any{
"sub": d.id, "aud": policyData.Identity.AudTunnel, "htm": policyData.Protocol.ProofHTM, "htu": e.srv.nodeURL, "iat": e.now().Unix(),
"jti": randomHex(12), "ath": sha256B64([]byte(session)), "nonce": nonce,
}
if mutate != nil {
mutate(header, claims)
}
return signJWS(d.key, header, claims)
}
// ---------------------------------------------------------------------------------------------------------------------------- device side
// dialNode connects to the node's own name the way a device does: TLS 1.3, the node's public certificate, ALPN all-tunnel/2.
func (e *testEnv) dialNode() (*tls.Conn, error) {
return tls.DialWithDialer(&net.Dialer{Timeout: 3 * time.Second}, "tcp", e.httpsAddr, &tls.Config{
ServerName: testNodeHost, RootCAs: e.nodePool, NextProtos: []string{policyData.Protocol.ALPN}, MinVersion: tls.VersionTLS13,
})
}
// devConn is the control connection of a device, played by a test.
type devConn struct {
t *testing.T
e *testEnv
d *testDevice
c *tls.Conn
br *bufio.Reader
session string
ok message // the parsed OK, once the handshake finished
feed chan string // once a devServer reads the connection, the lines that are not OPEN arrive here (closed at the end of the connection)
}
func (e *testEnv) connect(d *testDevice) *devConn {
e.t.Helper()
c, err := e.dialNode()
if err != nil {
e.t.Fatal(err)
}
dc := &devConn{t: e.t, e: e, d: d, c: c, br: bufio.NewReaderSize(c, 1<<17)}
e.t.Cleanup(func() { _ = c.Close() })
return dc
}
func (dc *devConn) send(line string) {
dc.t.Helper()
_ = dc.c.SetWriteDeadline(time.Now().Add(3 * time.Second))
if _, err := io.WriteString(dc.c, line+"\n"); err != nil {
dc.t.Fatalf("send %.40q: %v", line, err)
}
}
// try reads one line within timeout.
func (dc *devConn) try(timeout time.Duration) (string, error) {
if dc.feed != nil {
select {
case line, ok := <-dc.feed:
if !ok {
return "", io.EOF
}
return line, nil
case <-time.After(timeout):
return "", &net.OpError{Op: "read", Err: os.ErrDeadlineExceeded}
}
}
_ = dc.c.SetReadDeadline(time.Now().Add(timeout))
b, err := dc.br.ReadString('\n')
if err != nil {
return strings.TrimSuffix(b, "\n"), err
}
return strings.TrimSuffix(b, "\n"), nil
}
func (dc *devConn) recv() string {
dc.t.Helper()
line, err := dc.try(5 * time.Second)
if err != nil {
dc.t.Fatalf("no line from the node: %q %v", line, err)
}
return line
}
// expectClosed asserts the node closes the connection within timeout (any lines before the close are returned).
func (dc *devConn) expectClosed(timeout time.Duration) []string {
dc.t.Helper()
var lines []string
deadline := time.Now().Add(timeout)
for {
line, err := dc.try(time.Until(deadline))
if err != nil {
if ne, ok := err.(net.Error); ok && ne.Timeout() {
dc.t.Fatalf("connection still open after %s (lines so far %q)", timeout, lines)
}
return lines
}
lines = append(lines, line)
}
}
// hello performs HELLO and PROOF and returns the node's answer line (CHALLENGE and OK, or an ERR at any step) without judging it.
func (dc *devConn) hello(session func() string, proof func(session, nonce string) string) (final string) {
dc.t.Helper()
dc.session = session()
dc.send("HELLO 2 " + dc.session)
line := dc.recv()
m, err := parseLine(line, dirN2D)
if err != nil || m.str("verb") != "CHALLENGE" {
return line
}
dc.send("PROOF " + proof(dc.session, m.str("nonce")))
return dc.recv()
}
// connectOK completes the handshake for d with good tokens and fails the test unless the node says OK.
func (e *testEnv) connectOK(d *testDevice) *devConn {
e.t.Helper()
dc := e.connect(d)
line := dc.hello(func() string { return e.session(d, nil) }, func(s, n string) string { return e.proof(d, s, n, nil) })
m, err := parseLine(line, dirN2D)
if err != nil || m.str("verb") != "OK" {
e.t.Fatalf("handshake for %s: %q", d.id, line)
}
dc.ok = m
return dc
}
// helloAsync runs a whole HELLO for d on its own connection in the background and delivers the node's final line (or the error that ended the
// exchange); it never calls t.Fatal, so it is safe off the test goroutine.
func (e *testEnv) helloAsync(d *testDevice) <-chan string {
out := make(chan string, 1)
c, err := e.dialNode()
if err != nil {
e.t.Fatal(err)
}
e.t.Cleanup(func() { _ = c.Close() })
go func() {
br := bufio.NewReader(c)
read := func() string {
_ = c.SetReadDeadline(time.Now().Add(10 * time.Second))
l, err := br.ReadString('\n')
if err != nil {
return "read error: " + err.Error()
}
return strings.TrimSuffix(l, "\n")
}
session := e.session(d, nil)
_, _ = io.WriteString(c, "HELLO 2 "+session+"\n")
line := read()
if nonce, ok := strings.CutPrefix(line, "CHALLENGE "); ok {
_, _ = io.WriteString(c, "PROOF "+e.proof(d, session, nonce, nil)+"\n")
line = read()
}
out <- line
}()
return out
}
// renew sends RENEW with a fresh session, answers the CHALLENGE and returns the final line.
func (dc *devConn) renew() string {
dc.t.Helper()
dc.session = dc.e.session(dc.d, nil)
dc.send("RENEW " + dc.session)
line := dc.recv()
m, err := parseLine(line, dirN2D)
if err != nil || m.str("verb") != "CHALLENGE" {
return line
}
dc.send("PROOF " + dc.e.proof(dc.d, dc.session, m.str("nonce"), nil))
return dc.recv()
}
// devServer plays the device's side of the streams: it reads the control connection, attaches every OPEN through a new data connection and
// serves it with handler. Lines that are not OPEN are kept for the test.
type devServer struct {
dc *devConn
handler func(c net.Conn)
noAttach bool
mu sync.Mutex
opens []message
other []string
wg sync.WaitGroup
done chan struct{}
}
func (dc *devConn) serve(handler func(c net.Conn)) *devServer {
ds := &devServer{dc: dc, handler: handler, done: make(chan struct{})}
dc.feed = make(chan string, 1024)
if handler == nil {
ds.handler = ds.echo
}
go ds.loop()
return ds
}
func (ds *devServer) loop() {
defer close(ds.done)
defer close(ds.dc.feed)
for {
_ = ds.dc.c.SetReadDeadline(time.Time{})
line, err := ds.dc.br.ReadString('\n')
if err != nil {
return
}
line = strings.TrimSuffix(line, "\n")
m, perr := parseLine(line, dirN2D)
if perr == nil && m.str("verb") == "OPEN" {
ds.mu.Lock()
ds.opens = append(ds.opens, m)
ds.mu.Unlock()
if !ds.noAttach {
ds.wg.Add(1)
go ds.attach(m.str("stream_id"), m.str("secret"))
}
continue
}
ds.mu.Lock()
ds.other = append(ds.other, line)
ds.mu.Unlock()
select {
case ds.dc.feed <- line:
default:
}
}
}
func (ds *devServer) attach(id, secret string) {
defer ds.wg.Done()
c, err := ds.dc.e.dialNode()
if err != nil {
return
}
defer c.Close()
// No reply line: on success the raw stream starts right after ATTACH.
fmt.Fprintf(c, "ATTACH %s %s\n", id, secret)
ds.handler(c)
}
func (ds *devServer) openCount() int { ds.mu.Lock(); defer ds.mu.Unlock(); return len(ds.opens) }
func (ds *devServer) openList() []message {
ds.mu.Lock()
defer ds.mu.Unlock()
return append([]message(nil), ds.opens...)
}
func (ds *devServer) otherLines() []string {
ds.mu.Lock()
defer ds.mu.Unlock()
return append([]string(nil), ds.other...)
}
// seenRequest is what the device observed after TLS termination (plaintext).
type seenRequest struct {
method, uri, host string
header http.Header
body []byte
}
var (
seenMu sync.Mutex
seenByDS = map[*devServer][]seenRequest{}
)
func (ds *devServer) seen() []seenRequest {
seenMu.Lock()
defer seenMu.Unlock()
return append([]seenRequest(nil), seenByDS[ds]...)
}
// echo is the default device behavior: terminate TLS with the device certificate and answer HTTP/1.1 requests with "echo:"+body.
func (ds *devServer) echo(c net.Conn) {
tc := tls.Server(c, &tls.Config{Certificates: []tls.Certificate{ds.dc.d.cert}, NextProtos: []string{"http/1.1"}, MinVersion: tls.VersionTLS12})
_ = tc.SetDeadline(time.Now().Add(10 * time.Second))
if err := tc.Handshake(); err != nil {
return
}
_ = tc.SetDeadline(time.Time{})
br := bufio.NewReader(tc)
for {
req, err := http.ReadRequest(br)
if err != nil {
return
}
body, _ := io.ReadAll(req.Body)
seenMu.Lock()
seenByDS[ds] = append(seenByDS[ds], seenRequest{method: req.Method, uri: req.RequestURI, host: req.Host, header: req.Header.Clone(), body: body})
seenMu.Unlock()
out := append([]byte("echo:"), body...)
fmt.Fprintf(tc, "HTTP/1.1 200 OK\r\nContent-Length: %d\r\n\r\n", len(out))
if _, err := tc.Write(out); err != nil {
return
}
if req.Close {
return
}
}
}
// ---------------------------------------------------------------------------------------------------------------------------- client side
// clientTLS dials :443 as ChatGPT would: normal TLS verifying the DEVICE's certificate for d's host.
func (e *testEnv) clientTLS(d *testDevice) (*tls.Conn, error) {
return tls.DialWithDialer(&net.Dialer{Timeout: 3 * time.Second}, "tcp", e.httpsAddr, &tls.Config{
ServerName: d.host, RootCAs: d.pool, NextProtos: []string{"http/1.1"}, MinVersion: tls.VersionTLS12,
})
}
// postOver sends one POST with the given body over c and returns the response body.
func postOver(c net.Conn, host string, body []byte) ([]byte, error) {
_ = c.SetDeadline(time.Now().Add(20 * time.Second))
req, _ := http.NewRequest("POST", "https://"+host+"/mcp", bytes.NewReader(body))
req.ContentLength = int64(len(body))
if err := req.Write(c); err != nil {
return nil, err
}
resp, err := http.ReadResponse(bufio.NewReader(c), req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
return io.ReadAll(resp.Body)
}
// rawExchange sends raw bytes to addr and returns everything read until EOF.
func rawExchange(t *testing.T, addr, request string) string {
t.Helper()
c, err := net.DialTimeout("tcp", addr, 3*time.Second)
if err != nil {
t.Fatal(err)
}
defer c.Close()
_ = c.SetDeadline(time.Now().Add(10 * time.Second))
if _, err := io.WriteString(c, request); err != nil {
t.Fatal(err)
}
b, _ := io.ReadAll(c)
return string(b)
}
func eventually(t *testing.T, what string, within time.Duration, cond func() bool) {
t.Helper()
deadline := time.Now().Add(within)
for time.Now().Before(deadline) {
if cond() {
return
}
time.Sleep(10 * time.Millisecond)
}
t.Fatalf("timed out waiting for %s", what)
}
func (e *testEnv) waitIdle() {
e.t.Helper()
eventually(e.t, "all streams closed", 5*time.Second, func() bool { return e.srv.streams.Load() == 0 })
}
// chunkConn writes in tiny pieces with a pause, so the node sees a ClientHello arriving over many TCP reads.
type chunkConn struct {
net.Conn
size int
delay time.Duration
}
func (c chunkConn) Write(p []byte) (int, error) {
total := 0
for len(p) > 0 {
n := min(c.size, len(p))
w, err := c.Conn.Write(p[:n])
total += w
if err != nil {
return total, err
}
p = p[n:]
time.Sleep(c.delay)
}
return total, nil
}
// ---------------------------------------------------------------------------------------------------------------------------- the transcript
// transcriptDoc is the recording of the node in the contracts' format, closed with an end event, ready for ctCheckTranscript.
func (e *testEnv) transcriptDoc() map[string]any {
e.traceMu.Lock()
events := append([]any(nil), e.events...)
e.traceMu.Unlock()
events = append(events, map[string]any{"t": epochSeconds(e.now()), "ev": "end"})
keys := map[string]any{}
list := []any{}
if k, ok := e.keys.Key(testKID); ok {
keys = map[string]any{"kty": "OKP", "crv": "Ed25519", "x": b64Encode(k), "kid": testKID, "alg": "EdDSA", "use": "sig", "key_ops": []any{"verify"}}
list = append(list, keys)
}
return map[string]any{
"ctx": map[string]any{"node_id": testNodeID, "node_url": e.srv.nodeURL, "skew": float64(e.srv.cfg.ClockSkew / time.Second),
"jwks": map[string]any{"keys": list}, "randoms": map[string]any{}},
"events": events,
}
}
// checkTranscript runs the contracts' checker over what the node did and fails the test on the first violation.
func (e *testEnv) checkTranscript() {
e.t.Helper()
if violation := ctCheckTranscript(e.transcriptDoc()); violation != "" {
e.t.Fatalf("the node broke the contract: %s", violation)
}
}
// pemOf is the PEM of a certificate chain and its ECDSA key.
func pemOf(t *testing.T, cert tls.Certificate) (certPEM, keyPEM []byte) {
t.Helper()
for _, der := range cert.Certificate {
certPEM = append(certPEM, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})...)
}
der, err := x509.MarshalECPrivateKey(cert.PrivateKey.(*ecdsa.PrivateKey))
if err != nil {
t.Fatal(err)
}
return certPEM, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: der})
}
files/tunnel/hints.go 120 lines · 3.2 KB
package main
import (
"net/netip"
"sync"
"time"
)
// HINT (the design plan, sections 2 and 5): a device tells its node which source addresses just completed authenticated calls to it. They get the
// slice of the device's stream cap that anonymous sources cannot use (policy node_limits.label_hint_slice), so a flood from many addresses cannot lock
// out the one that is ChatGPT. The table is memory only, an address stays good for hint_ttl_s after the last HINT that named it, a device can mark
// at most hint_addresses_per_label addresses (the oldest go first), and a mark is filed under the host the control plane's answer gave that connection:
// a device cannot mark an address for any host but its own.
type hintSet struct {
until map[netip.Addr]time.Time
}
// hintTable is the node's record of the addresses devices marked good, per device host.
type hintTable struct {
clk clock
ttl time.Duration
cap int
mu sync.Mutex
hosts map[string]*hintSet
}
func newHintTable(clk clock, ttl time.Duration, perLabel int) *hintTable {
return &hintTable{clk: clk, ttl: ttl, cap: perLabel, hosts: map[string]*hintSet{}}
}
// mark records the addresses (never more than the protocol's list) as good for host from now until the TTL.
func (h *hintTable) mark(host string, addrs []netip.Addr) {
now := h.clk.Now()
h.mu.Lock()
defer h.mu.Unlock()
set := h.hosts[host]
if set == nil {
set = &hintSet{until: map[netip.Addr]time.Time{}}
h.hosts[host] = set
}
for _, a := range addrs {
set.until[a.WithZone("").Unmap()] = now.Add(h.ttl)
}
if len(set.until) <= h.cap {
return
}
for a, t := range set.until { // expired first
if !now.Before(t) {
delete(set.until, a)
}
}
for len(set.until) > h.cap { // then the ones that will expire soonest, which are the ones marked longest ago
var oldest netip.Addr
var first time.Time
for a, t := range set.until {
if first.IsZero() || t.Before(first) {
oldest, first = a, t
}
}
delete(set.until, oldest)
}
}
// good reports whether the device of host marked addr and the mark has not expired.
func (h *hintTable) good(host string, addr netip.Addr) bool {
now := h.clk.Now()
addr = addr.WithZone("").Unmap()
h.mu.Lock()
defer h.mu.Unlock()
set := h.hosts[host]
if set == nil {
return false
}
t, ok := set.until[addr]
return ok && now.Before(t)
}
// goodAnywhere reports whether the device of ANY host marked addr and the mark has not expired.
func (h *hintTable) goodAnywhere(addr netip.Addr) bool {
now := h.clk.Now()
addr = addr.WithZone("").Unmap()
h.mu.Lock()
defer h.mu.Unlock()
for _, set := range h.hosts {
if t, ok := set.until[addr]; ok && now.Before(t) {
return true
}
}
return false
}
// Sweep forgets what expired (a background task calls it).
func (h *hintTable) Sweep() {
now := h.clk.Now()
h.mu.Lock()
defer h.mu.Unlock()
for host, set := range h.hosts {
for a, t := range set.until {
if !now.Before(t) {
delete(set.until, a)
}
}
if len(set.until) == 0 {
delete(h.hosts, host)
}
}
}
// size is the number of addresses held, for the metrics (an aggregate: never per device).
func (h *hintTable) size() int {
h.mu.Lock()
defer h.mu.Unlock()
n := 0
for _, set := range h.hosts {
n += len(set.until)
}
return n
}
files/tunnel/hints_test.go 93 lines · 2.6 KB
package main
import (
"fmt"
"net/netip"
"testing"
"time"
)
func newTestHints(ttl time.Duration, perLabel int) (*hintTable, *fakeClock) {
fc := newFakeClock()
return newHintTable(newClock(fc.Now, fc.AfterFunc), ttl, perLabel), fc
}
func addrs(texts ...string) []netip.Addr {
var out []netip.Addr
for _, t := range texts {
out = append(out, netip.MustParseAddr(t))
}
return out
}
func TestHints_AMarkLastsItsTTLFromTheLastHintAndBelongsToOneHost(t *testing.T) {
h, fc := newTestHints(15*time.Minute, 64)
a := netip.MustParseAddr("203.0.113.5")
h.mark("one.example", addrs("203.0.113.5"))
if !h.good("one.example", a) {
t.Fatal("not good right after the HINT")
}
if h.good("two.example", a) {
t.Fatal("a mark for one host counts for another")
}
fc.Advance(14 * time.Minute)
if !h.good("one.example", a) {
t.Fatal("expired early")
}
h.mark("one.example", addrs("203.0.113.5")) // the device says it again: the clock starts over
fc.Advance(14 * time.Minute)
if !h.good("one.example", a) {
t.Fatal("a repeated HINT did not renew the mark")
}
fc.Advance(2 * time.Minute)
if h.good("one.example", a) {
t.Fatal("outlived its TTL")
}
h.Sweep()
if h.size() != 0 {
t.Fatalf("%d left after the sweep", h.size())
}
}
func TestHints_AddressesAreCompared_AsTheyAre_WithoutZoneOrMapping(t *testing.T) {
h, _ := newTestHints(time.Hour, 64)
h.mark("h", addrs("::ffff:203.0.113.9", "fe80::1%eth0"))
if !h.good("h", netip.MustParseAddr("203.0.113.9")) {
t.Fatal("an IPv4-mapped mark is the IPv4 address")
}
if !h.good("h", netip.MustParseAddr("fe80::1")) {
t.Fatal("a zone is not part of an address")
}
if h.good("h", netip.MustParseAddr("203.0.113.10")) {
t.Fatal("the neighbouring address is good too")
}
}
func TestHints_ADeviceCanMarkOnlyAFixedNumberAndTheOldestGoFirst(t *testing.T) {
h, fc := newTestHints(time.Hour, 5)
for i := 0; i < 5; i++ {
h.mark("h", addrs(fmt.Sprintf("203.0.113.%d", i+1)))
fc.Advance(time.Second)
}
h.mark("h", addrs("203.0.113.100", "203.0.113.101"))
if h.size() != 5 {
t.Fatalf("%d addresses, cap 5", h.size())
}
for _, gone := range []string{"203.0.113.1", "203.0.113.2"} {
if h.good("h", netip.MustParseAddr(gone)) {
t.Fatalf("%s kept although it is the oldest", gone)
}
}
for _, kept := range []string{"203.0.113.3", "203.0.113.5", "203.0.113.100", "203.0.113.101"} {
if !h.good("h", netip.MustParseAddr(kept)) {
t.Fatalf("%s lost", kept)
}
}
// a flood of hints cannot grow the table without bound
for i := 0; i < 1000; i++ {
h.mark("h", addrs(fmt.Sprintf("198.51.100.%d", i%250)))
}
if h.size() > 5 {
t.Fatalf("%d addresses after a flood", h.size())
}
}
files/tunnel/jsonparse.go 187 lines · 4.2 KB
package main
import (
"encoding/json"
"errors"
"fmt"
"io"
"math"
"sort"
"strconv"
"strings"
"unicode/utf8"
)
// The tokens and bodies of the contracts are JSON that every implementation must read the same way, so the node reads it strictly: valid
// UTF-8, no repeated key (a body with a repeated key means different things to different parsers), no non-integral number, nothing after the value.
// parseJSON parses one JSON value into maps, slices, strings, bools, nil and float64 (integers up to 2^53 are exact).
func parseJSON(text string) (any, error) {
if !utf8.ValidString(text) {
return nil, errors.New("utf-8")
}
dec := json.NewDecoder(strings.NewReader(text))
dec.UseNumber()
value, err := decodeValue(dec)
if err != nil {
return nil, err
}
if _, err := dec.Token(); err != io.EOF {
return nil, errors.New("trailing data")
}
if err := noFractions(value); err != nil {
return nil, err
}
return value, nil
}
func decodeValue(dec *json.Decoder) (any, error) {
token, err := dec.Token()
if err != nil {
return nil, err
}
switch t := token.(type) {
case json.Delim:
switch t {
case '{':
out := map[string]any{}
for dec.More() {
keyToken, err := dec.Token()
if err != nil {
return nil, err
}
key, ok := keyToken.(string)
if !ok {
return nil, errors.New("key")
}
if _, dup := out[key]; dup {
return nil, errors.New("duplicate key")
}
value, err := decodeValue(dec)
if err != nil {
return nil, err
}
out[key] = value
}
if _, err := dec.Token(); err != nil {
return nil, err
}
return out, nil
case '[':
out := []any{}
for dec.More() {
value, err := decodeValue(dec)
if err != nil {
return nil, err
}
out = append(out, value)
}
if _, err := dec.Token(); err != nil {
return nil, err
}
return out, nil
}
return nil, errors.New("delimiter")
case json.Number:
f, err := strconv.ParseFloat(string(t), 64)
if err != nil || math.IsInf(f, 0) {
return nil, errors.New("number")
}
return f, nil
default:
return token, nil // string, bool, nil
}
}
// noFractions refuses a non-integral number anywhere in a value (the contracts' bodies hold integers, never fractions).
func noFractions(value any) error {
switch v := value.(type) {
case float64:
if v != math.Trunc(v) {
return errors.New("fraction")
}
case map[string]any:
for _, item := range v {
if err := noFractions(item); err != nil {
return err
}
}
case []any:
for _, item := range v {
if err := noFractions(item); err != nil {
return err
}
}
}
return nil
}
func escapeJSON(s string) string {
var b strings.Builder
b.WriteByte('"')
for _, r := range s {
switch {
case r == '"':
b.WriteString(`\"`)
case r == '\\':
b.WriteString(`\\`)
case r == '\n':
b.WriteString(`\n`)
case r == '\r':
b.WriteString(`\r`)
case r == '\t':
b.WriteString(`\t`)
case r == '\b':
b.WriteString(`\b`)
case r == '\f':
b.WriteString(`\f`)
case r < 0x20 || r > 0x7e:
if r > 0xffff {
r -= 0x10000
fmt.Fprintf(&b, `\u%04x\u%04x`, 0xd800+(r>>10), 0xdc00+(r&0x3ff))
} else {
fmt.Fprintf(&b, `\u%04x`, r)
}
default:
b.WriteRune(r)
}
}
b.WriteByte('"')
return b.String()
}
// canonicalJSON is the form of a JSON body on a wire line: keys sorted, no whitespace, ASCII only, integers without a fraction.
func canonicalJSON(value any) string {
switch v := value.(type) {
case nil:
return "null"
case bool:
return strconv.FormatBool(v)
case int:
return strconv.Itoa(v)
case int64:
return strconv.FormatInt(v, 10)
case float64:
return strconv.FormatInt(int64(v), 10)
case string:
return escapeJSON(v)
case []any:
parts := make([]string, len(v))
for i, item := range v {
parts[i] = canonicalJSON(item)
}
return "[" + strings.Join(parts, ",") + "]"
case map[string]any:
keys := make([]string, 0, len(v))
for key := range v {
keys = append(keys, key)
}
sort.Strings(keys)
parts := make([]string, len(keys))
for i, key := range keys {
parts[i] = escapeJSON(key) + ":" + canonicalJSON(v[key])
}
return "{" + strings.Join(parts, ",") + "}"
}
panic(fmt.Sprintf("canonical json: %T", value))
}
files/tunnel/jwks.go 335 lines · 10.8 KB
package main
import (
"context"
"crypto/ed25519"
"encoding/base64"
"errors"
"log/slog"
"regexp"
"sync"
"time"
)
// The control plane's published keys are the root of trust for the sessions a node verifies offline. The node reads them through KeySource: "is this
// kid a key of the control plane, and which one". What fetches, caches and refreshes them is JWKSCache below (issuer pinned, 300 s cache, reload on an
// unknown kid, at most 16 keys, fail closed after 30 minutes without a refresh).
// KeySource gives the public Ed25519 key of a control-plane kid. A source that cannot vouch for its keys any more (stale beyond the policy's window)
// answers false for every kid: the node then refuses every session, it never trusts an old set.
type KeySource interface {
Key(kid string) (ed25519.PublicKey, bool)
}
// keyHealth is what a KeySource may add: whether it can vouch for its keys right now. When it cannot, a session whose kid it does not know is refused
// as control_unavailable (try again later) and not as auth (your credentials are wrong).
type keyHealth interface {
Healthy() bool
}
// StaticKeys is a fixed key set: what the tests and the test-node binary use, and what a real source returns once it has a fresh set.
type StaticKeys map[string]ed25519.PublicKey
func (k StaticKeys) Key(kid string) (ed25519.PublicKey, bool) {
key, ok := k[kid]
return key, ok
}
// SwapKeys is a key set that can be replaced while the node runs (a rotation, or a refresh that returned a new set). It is safe for concurrent use.
type SwapKeys struct {
mu sync.RWMutex
keys StaticKeys
}
func NewSwapKeys(keys StaticKeys) *SwapKeys { return &SwapKeys{keys: keys} }
// Set replaces the whole set; a nil or empty set refuses everything.
func (s *SwapKeys) Set(keys StaticKeys) {
s.mu.Lock()
s.keys = keys
s.mu.Unlock()
}
func (s *SwapKeys) Key(kid string) (ed25519.PublicKey, bool) {
s.mu.RLock()
defer s.mu.RUnlock()
return s.keys.Key(kid)
}
var errBadJWKS = errors.New("bad_jwks")
var reB64Charset = regexp.MustCompile(`^[A-Za-z0-9_-]+$`)
// b64Decode is the strict base64url of the contracts: no padding and the canonical encoding (a text that decodes to bytes that encode differently
// is refused, so one token has one text).
func b64Decode(text string) ([]byte, error) {
if !reB64Charset.MatchString(text) {
return nil, errors.New("base64url")
}
raw, err := base64.RawURLEncoding.Strict().DecodeString(text)
if err != nil || base64.RawURLEncoding.EncodeToString(raw) != text {
return nil, errors.New("base64url")
}
return raw, nil
}
func b64Encode(raw []byte) string { return base64.RawURLEncoding.EncodeToString(raw) }
// publicKeyX decodes the x of an Ed25519 JWK: 43 canonical base64url characters that are 32 bytes.
func publicKeyX(x string) (ed25519.PublicKey, bool) {
if len(x) != 43 {
return nil, false
}
raw, err := b64Decode(x)
if err != nil || len(raw) != ed25519.PublicKeySize {
return nil, false
}
return ed25519.PublicKey(raw), true
}
// ParseJWKS reads the control plane's GET /.well-known/device-jwks.json (jwks.schema.json): one to max_keys Ed25519 keys, each with its kid, alg
// EdDSA, use sig and key_ops [verify], no private part, no repeated kid. A set the schema refuses is bad_jwks and nothing of it is used.
func ParseJWKS(doc []byte) (StaticKeys, error) {
value, err := parseJSON(string(doc))
if err != nil {
return nil, errBadJWKS
}
top, ok := value.(map[string]any)
if !ok || len(top) != 1 {
return nil, errBadJWKS
}
list, ok := top["keys"].([]any)
if !ok || len(list) < 1 || len(list) > policyData.JWKS.MaxKeys {
return nil, errBadJWKS
}
keys := StaticKeys{}
for _, item := range list {
m, ok := item.(map[string]any)
if !ok || len(m) != 7 {
return nil, errBadJWKS
}
kid, _ := m["kid"].(string)
x, _ := m["x"].(string)
ops, _ := m["key_ops"].([]any)
if m["kty"] != "OKP" || m["crv"] != "Ed25519" || m["alg"] != "EdDSA" || m["use"] != "sig" || !reKID.MatchString(kid) ||
len(ops) != 1 || ops[0] != "verify" {
return nil, errBadJWKS
}
key, ok := publicKeyX(x)
if !ok {
return nil, errBadJWKS
}
if _, dup := keys[kid]; dup {
return nil, errBadJWKS
}
keys[kid] = key
}
return keys, nil
}
// ---------------------------------------------------------------------------------------------------------------------------- the cache
// JWKSConfig configures a JWKSCache. Every window defaults to the policy's.
type JWKSConfig struct {
// Fetch reads the control plane's GET /.well-known/device-jwks.json. The URL is the configured control plane's and nothing else: the issuer is
// pinned by the caller, an answer cannot name another place to read the keys from.
Fetch func(ctx context.Context) ([]byte, error)
Clock clock
TTL time.Duration // how long a fetched set is used before the next refresh
StaleAfter time.Duration // beyond this without a successful refresh the cache vouches for nothing
MinReload time.Duration // between two fetches that an unknown kid (or a stale set) asked for: a flood of random kids is one fetch, not thousands
FetchTimeout time.Duration
Observe func(ok bool) // every fetch's outcome (the node's reachability record)
Log *slog.Logger
}
// JWKSCache is the node's KeySource for the control plane's keys. It fetches the set at start and every TTL (a failure is retried with the policy's
// backoff and the old set keeps serving), reloads when a session names a kid it does not hold (so a rotation reaches a node at the next RENEW, with at
// most one fetch per MinReload however many sessions ask), keeps at most the policy's number of keys (ParseJWKS refuses a bigger set whole), replaces
// the whole set on every successful fetch (a key the control plane dropped stops being trusted), and FAILS CLOSED: after StaleAfter without a refresh
// it answers false for every kid, an old set is never trusted.
type JWKSCache struct {
cfg JWKSConfig
mu sync.Mutex
keys StaticKeys
fetchedAt time.Time // the last successful fetch
attemptedAt time.Time // the last fetch of any outcome that was asked for by a lookup
flight chan struct{}
failures int
}
func NewJWKSCache(cfg JWKSConfig) *JWKSCache {
if cfg.TTL <= 0 {
cfg.TTL = secs(policyData.JWKS.CacheS)
}
if cfg.StaleAfter <= 0 {
cfg.StaleAfter = secs(policyData.JWKS.StaleFailClosedS)
}
if cfg.MinReload <= 0 {
cfg.MinReload = minReload
}
if cfg.FetchTimeout <= 0 {
cfg.FetchTimeout = fetchTimeout
}
if cfg.Clock.now == nil {
cfg.Clock = newClock(nil, nil)
}
if cfg.Log == nil {
cfg.Log = slog.New(slog.DiscardHandler)
}
return &JWKSCache{cfg: cfg}
}
// fetchTimeout bounds one fetch of the keys (a lookup that triggers one waits at most this long).
const fetchTimeout = 5 * time.Second
// minReload is the default gap between fetches that lookups ask for.
const minReload = 10 * time.Second
func (c *JWKSCache) fresh(now time.Time) bool {
return !c.fetchedAt.IsZero() && now.Sub(c.fetchedAt) <= c.cfg.StaleAfter
}
// Healthy reports whether the cache can vouch for its keys: it has a set that is not older than StaleAfter.
func (c *JWKSCache) Healthy() bool {
c.mu.Lock()
defer c.mu.Unlock()
return c.fresh(c.cfg.Clock.Now())
}
// Age is how long ago the last successful fetch was (a negative duration when there has not been one): the monitor's "age of the JWKS".
func (c *JWKSCache) Age() time.Duration {
c.mu.Lock()
defer c.mu.Unlock()
if c.fetchedAt.IsZero() {
return -1
}
return c.cfg.Clock.Now().Sub(c.fetchedAt)
}
// Key is the KeySource lookup.
func (c *JWKSCache) Key(kid string) (ed25519.PublicKey, bool) {
now := c.cfg.Clock.Now()
c.mu.Lock()
fresh := c.fresh(now)
key, known := c.keys[kid]
due := !c.fetchedAt.IsZero() && now.Sub(c.fetchedAt) >= c.cfg.TTL
askAllowed := c.attemptedAt.IsZero() || now.Sub(c.attemptedAt) >= c.cfg.MinReload
if due && fresh && known && askAllowed {
// The attempt is stamped here, with the reading this lookup took, not by the goroutine whenever it gets to run: the gap to the next attempt
// must not depend on scheduling, and a burst of lookups past the TTL starts one refresh, not one each.
c.attemptedAt = now
}
c.mu.Unlock()
switch {
case fresh && known && !due:
return key, true
case fresh && known:
// Past its TTL but still vouched for: serve it and refresh behind the lookup (the periodic loop is the main path; this covers a busy node
// whose loop is in a backoff).
if askAllowed {
go func() { _ = c.reload(context.Background(), false) }()
}
return key, true
case !askAllowed:
return nil, false
}
// An unknown kid, or a set that is no longer vouched for: one fetch (every concurrent lookup waits for the same one), then the answer.
_ = c.reload(context.Background(), true)
c.mu.Lock()
defer c.mu.Unlock()
if !c.fresh(c.cfg.Clock.Now()) {
return nil, false
}
key, known = c.keys[kid]
return key, known
}
// Refresh fetches the set now (single flight) and replaces the cached one when the answer is a valid set. It returns the fetch's error.
func (c *JWKSCache) Refresh(ctx context.Context) error { return c.reload(ctx, false) }
func (c *JWKSCache) reload(ctx context.Context, fromLookup bool) error {
c.mu.Lock()
if wait := c.flight; wait != nil {
c.mu.Unlock()
select {
case <-wait:
case <-ctx.Done():
return ctx.Err()
}
c.mu.Lock()
defer c.mu.Unlock()
if c.fresh(c.cfg.Clock.Now()) {
return nil
}
return errors.New("jwks refresh failed")
}
wait := make(chan struct{})
c.flight = wait
if fromLookup {
c.attemptedAt = c.cfg.Clock.Now()
}
c.mu.Unlock()
fctx, cancel := context.WithTimeout(ctx, c.cfg.FetchTimeout)
doc, err := c.cfg.Fetch(fctx)
cancel()
var keys StaticKeys
if err == nil {
keys, err = ParseJWKS(doc)
}
now := c.cfg.Clock.Now()
c.mu.Lock()
if err == nil {
c.keys, c.fetchedAt, c.failures = keys, now, 0
} else {
c.failures++
}
c.flight = nil
close(wait)
c.mu.Unlock()
if c.cfg.Observe != nil {
c.cfg.Observe(err == nil)
}
if err != nil {
c.cfg.Log.Warn("jwks_refresh_failed", "reason", errorWord(err))
}
return err
}
// errorWord is a short, address-free word for a failure: nothing from the network goes to a log.
func errorWord(err error) string {
switch {
case errors.Is(err, errBadJWKS):
return "bad_jwks"
case errors.Is(err, context.DeadlineExceeded):
return "timeout"
case errors.Is(err, context.Canceled):
return "canceled"
}
var ce *ControlError
if errors.As(err, &ce) {
return ce.Code
}
return "unreachable"
}
// Run keeps the set fresh until ctx ends: a fetch now, then one every TTL (spread a little, so two nodes do not call together), and after a failure the
// policy's backoff.
func (c *JWKSCache) Run(ctx context.Context) {
for {
err := c.Refresh(ctx)
c.mu.Lock()
failures := c.failures
c.mu.Unlock()
wait := jitter(c.cfg.TTL, 10)
if err != nil {
wait = backoff(failures)
}
if !c.cfg.Clock.sleep(ctx, wait) {
return
}
}
}
files/tunnel/jwks_test.go 484 lines · 16.7 KB
package main
import (
"bufio"
"context"
"crypto/ed25519"
"crypto/rand"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
)
// The JWKS cache: the control plane's keys are the root of trust of every session the node verifies offline.
// jwksSource is a control plane's key endpoint as the cache sees it: a set that the tests change and a switch that makes it fail.
type jwksSource struct {
mu sync.Mutex
keys map[string]ed25519.PublicKey
fail error
raw []byte
calls atomic.Int64
}
func newJWKSSource(kids ...string) *jwksSource {
s := &jwksSource{keys: map[string]ed25519.PublicKey{}}
for _, kid := range kids {
s.add(kid)
}
return s
}
func (s *jwksSource) add(kid string) ed25519.PublicKey {
pub, _, _ := ed25519.GenerateKey(rand.Reader)
s.mu.Lock()
s.keys[kid] = pub
s.mu.Unlock()
return pub
}
func (s *jwksSource) drop(kid string) {
s.mu.Lock()
delete(s.keys, kid)
s.mu.Unlock()
}
func (s *jwksSource) setFail(err error) {
s.mu.Lock()
s.fail = err
s.mu.Unlock()
}
func (s *jwksSource) fetch(ctx context.Context) ([]byte, error) {
s.calls.Add(1)
s.mu.Lock()
defer s.mu.Unlock()
if s.fail != nil {
return nil, s.fail
}
if s.raw != nil {
return s.raw, nil
}
list := []any{}
for kid, key := range s.keys {
list = append(list, map[string]any{"kty": "OKP", "crv": "Ed25519", "x": b64Encode(key), "kid": kid, "alg": "EdDSA", "use": "sig", "key_ops": []any{"verify"}})
}
return []byte(canonicalJSON(map[string]any{"keys": list})), nil
}
func newTestCache(src *jwksSource, clk *fakeClock) *JWKSCache {
return NewJWKSCache(JWKSConfig{Fetch: src.fetch, Clock: newClock(clk.Now, clk.AfterFunc), MinReload: 5 * time.Second})
}
func TestJWKS_FetchesOnDemandCachesFor300sAndRefreshesBehindTheLookup(t *testing.T) {
src := newJWKSSource("a")
clk := newFakeClock()
c := newTestCache(src, clk)
if policyData.JWKS.CacheS != 300 || c.cfg.TTL != 300*time.Second {
t.Fatalf("the cache lives %s; the plan says 300 s", c.cfg.TTL)
}
if _, ok := c.Key("a"); !ok || src.calls.Load() != 1 {
t.Fatalf("first lookup: fetched %d", src.calls.Load())
}
for i := 0; i < 50; i++ {
if _, ok := c.Key("a"); !ok {
t.Fatal("a cached key was lost")
}
}
if src.calls.Load() != 1 {
t.Fatalf("a cached set was fetched again: %d", src.calls.Load())
}
// past its TTL a lookup still answers from the set (it is vouched for) and refreshes behind it
clk.Advance(301 * time.Second)
if _, ok := c.Key("a"); !ok {
t.Fatal("a set past its TTL but inside the stale window must still serve")
}
eventually(t, "the refresh behind the lookup", 2*time.Second, func() bool { return src.calls.Load() == 2 })
}
func TestJWKS_ARotationIsFollowedAtTheNextLookupAndADroppedKeyStopsBeingTrusted(t *testing.T) {
src := newJWKSSource("old")
clk := newFakeClock()
c := newTestCache(src, clk)
if _, ok := c.Key("old"); !ok {
t.Fatal("old")
}
newPub := src.add("new")
clk.Advance(6 * time.Second) // past the minimum gap between forced reloads
got, ok := c.Key("new") // an unknown kid reloads
if !ok || !got.Equal(newPub) {
t.Fatal("a rotated-in key was not picked up at the first lookup that needed it")
}
if _, ok := c.Key("old"); !ok {
t.Fatal("the old key is still published, so still trusted")
}
src.drop("old")
clk.Advance(301 * time.Second)
if err := c.Refresh(context.Background()); err != nil {
t.Fatal(err)
}
if _, ok := c.Key("old"); ok {
t.Fatal("a key the control plane stopped publishing is still trusted")
}
if _, ok := c.Key("new"); !ok {
t.Fatal("new")
}
}
func TestJWKS_AFloodOfUnknownKidsIsOneFetchPerMinimumGapNotAFetchPerSession(t *testing.T) {
src := newJWKSSource("a")
clk := newFakeClock()
c := newTestCache(src, clk)
if _, ok := c.Key("a"); !ok {
t.Fatal("a")
}
before := src.calls.Load()
var wg sync.WaitGroup
var wrongly atomic.Int64
for i := 0; i < 4000; i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
if _, ok := c.Key(fmt.Sprintf("random-%d", i)); ok {
wrongly.Add(1)
}
}(i)
}
wg.Wait()
if wrongly.Load() != 0 {
t.Fatal("an unknown kid was accepted")
}
// the first of them may fetch (the cache had not looked for a while); the rest never do
if n := src.calls.Load() - before; n > 1 {
t.Fatalf("4000 sessions with unknown kids made %d fetches", n)
}
// a legitimate rotation right after the flood waits for the gap, and is picked up after it
src.add("rotated")
if _, ok := c.Key("rotated"); ok {
t.Fatal("served inside the minimum gap")
}
clk.Advance(6 * time.Second)
if _, ok := c.Key("rotated"); !ok {
t.Fatal("not picked up after the gap")
}
// and the known keys were never refused while the flood ran
if _, ok := c.Key("a"); !ok {
t.Fatal("a known key was refused")
}
}
// waitIdle waits until the source has been asked n times and no fetch is in flight.
func (c *JWKSCache) waitIdle(t *testing.T, src *jwksSource, n int64) {
t.Helper()
eventually(t, "the background refresh to end", 3*time.Second, func() bool {
c.mu.Lock()
defer c.mu.Unlock()
return src.calls.Load() >= n && c.flight == nil
})
}
func TestJWKS_ALookupPastTheTTLStampsItsRefreshBeforeItReturns(t *testing.T) {
src := newJWKSSource("a")
clk := newFakeClock()
c := newTestCache(src, clk)
if _, ok := c.Key("a"); !ok {
t.Fatal("start")
}
clk.Advance(6 * time.Minute) // past the 5 minute TTL, still vouched for
at := clk.Now()
if _, ok := c.Key("a"); !ok {
t.Fatal("a set past its TTL but not stale must still be served")
}
// the gap to the next attempt counts from this lookup, whenever the refresh goroutine happens to run
c.mu.Lock()
stamped := c.attemptedAt
c.mu.Unlock()
if !stamped.Equal(at) {
t.Fatalf("the attempt is stamped %s, the lookup was at %s", stamped, at)
}
c.waitIdle(t, src, 2)
if _, ok := c.Key("zzz"); ok || src.calls.Load() != 2 {
t.Fatalf("an unknown kid inside the gap fetched again (%d fetches)", src.calls.Load())
}
}
func TestJWKS_FailsClosedAfterThirtyMinutesWithoutARefreshAndRecovers(t *testing.T) {
if policyData.JWKS.StaleFailClosedS != 1800 {
t.Fatalf("the plan says 30 minutes, the policy says %d s", policyData.JWKS.StaleFailClosedS)
}
src := newJWKSSource("a")
clk := newFakeClock()
c := newTestCache(src, clk)
if _, ok := c.Key("a"); !ok || !c.Healthy() {
t.Fatal("start")
}
src.setFail(errors.New("unreachable"))
// the control plane is down: the set keeps serving for the stale window
clk.Advance(29 * time.Minute)
if _, ok := c.Key("a"); !ok || !c.Healthy() {
t.Fatal("a set 29 minutes old must still be trusted")
}
// that lookup refreshed the set behind itself (and failed): the fake clock moves only once it is over
c.waitIdle(t, src, 2)
clk.Advance(2 * time.Minute) // 31 minutes without a refresh
if _, ok := c.Key("a"); ok {
t.Fatal("a set 31 minutes old was trusted: the cache must fail closed")
}
if c.Healthy() {
t.Fatal("healthy while stale")
}
if age := c.Age(); age < 31*time.Minute {
t.Fatalf("age %s", age)
}
// the control plane is back: the next lookup (after the gap) recovers by itself
src.setFail(nil)
clk.Advance(6 * time.Second)
if _, ok := c.Key("a"); !ok || !c.Healthy() {
t.Fatal("the cache did not recover when the control plane came back")
}
}
func TestJWKS_ABadSetIsNeverUsedAndNeverReplacesAGoodOne(t *testing.T) {
src := newJWKSSource("good")
clk := newFakeClock()
c := newTestCache(src, clk)
if _, ok := c.Key("good"); !ok {
t.Fatal("start")
}
pub, _, _ := ed25519.GenerateKey(rand.Reader)
x := b64Encode(pub)
key := func(kid string) string {
return fmt.Sprintf(`{"kty":"OKP","crv":"Ed25519","x":%q,"kid":%q,"alg":"EdDSA","use":"sig","key_ops":["verify"]}`, x, kid)
}
many := make([]string, 17)
for i := range many {
many[i] = key(fmt.Sprintf("k%d", i))
}
for name, raw := range map[string]string{
"17 keys": `{"keys":[` + strings.Join(many, ",") + `]}`,
"a repeated kid": `{"keys":[` + key("dup") + `,` + key("dup") + `]}`,
"no keys": `{"keys":[]}`,
"a private part": `{"keys":[{"kty":"OKP","crv":"Ed25519","x":"` + x + `","d":"` + x + `","kid":"p","alg":"EdDSA","use":"sig","key_ops":["verify"]}]}`,
"not json": `<html>`,
"an extra member": `{"keys":[` + key("a") + `],"issuer":"https://elsewhere.example"}`,
} {
src.mu.Lock()
src.raw = []byte(raw)
src.mu.Unlock()
clk.Advance(6 * time.Second)
if err := c.Refresh(context.Background()); err == nil {
t.Errorf("%s: accepted", name)
}
if _, ok := c.Key("good"); !ok {
t.Errorf("%s: the good set was lost", name)
}
if _, ok := c.Key("dup"); ok {
t.Errorf("%s: part of a refused set was used", name)
}
}
// sixteen keys is the most
sixteen := many[:16]
src.mu.Lock()
src.raw = []byte(`{"keys":[` + strings.Join(sixteen, ",") + `]}`)
src.mu.Unlock()
if err := c.Refresh(context.Background()); err != nil {
t.Fatalf("16 keys: %v", err)
}
if _, ok := c.Key("k15"); !ok {
t.Fatal("k15")
}
}
func TestJWKS_RunKeepsTheSetFreshAndBacksOffAfterAFailure(t *testing.T) {
src := newJWKSSource("a")
clk := newFakeClock()
var observed []bool
var mu sync.Mutex
c := NewJWKSCache(JWKSConfig{Fetch: src.fetch, Clock: newClock(clk.Now, clk.AfterFunc), Observe: func(ok bool) {
mu.Lock()
observed = append(observed, ok)
mu.Unlock()
}})
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() { c.Run(ctx); close(done) }()
eventually(t, "the first fetch", 2*time.Second, func() bool { return src.calls.Load() == 1 })
eventually(t, "the loop sleeping", 2*time.Second, func() bool { return clk.pending() == 1 })
clk.Advance(5 * time.Minute) // the TTL, spread by at most 10 percent: not yet at 270 s, surely at 330 s
clk.Advance(30 * time.Second)
eventually(t, "the periodic refresh", 2*time.Second, func() bool { return src.calls.Load() == 2 })
eventually(t, "the loop sleeping again", 2*time.Second, func() bool { return clk.pending() == 1 })
src.setFail(errors.New("down"))
clk.Advance(6 * time.Minute)
eventually(t, "a failed refresh", 2*time.Second, func() bool { return src.calls.Load() == 3 })
eventually(t, "the backoff timer", 2*time.Second, func() bool { return clk.pending() == 1 })
// after a failure the next try comes after the policy's backoff (1 s, then 2 s), not after another 5 minutes
clk.Advance(2 * time.Second)
eventually(t, "the retry after the backoff", 2*time.Second, func() bool { return src.calls.Load() == 4 })
cancel()
select {
case <-done:
case <-time.After(2 * time.Second):
t.Fatal("Run did not end with its context")
}
mu.Lock()
defer mu.Unlock()
if len(observed) < 4 || !observed[0] || observed[2] {
t.Fatalf("observed outcomes %v", observed)
}
}
func TestJWKS_ThePinnedControlPlaneIsTheOnlyPlaceTheKeysComeFrom(t *testing.T) {
var landed atomic.Int64
elsewhere := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
landed.Add(1)
_, _ = w.Write([]byte(`{"keys":[]}`))
}))
defer elsewhere.Close()
redirecting := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, elsewhere.URL+jwksPath, http.StatusFound)
}))
defer redirecting.Close()
_, priv, _ := ed25519.GenerateKey(rand.Reader)
hc, err := NewHTTPControl(HTTPControlConfig{BaseURL: redirecting.URL, NodeID: "t1", KID: "k1", Key: priv, Client: redirecting.Client()})
if err != nil {
t.Fatal(err)
}
if _, err := hc.FetchJWKS(context.Background()); err == nil || landed.Load() != 0 {
t.Fatalf("the keys were read through a redirect: err=%v landed=%d", err, landed.Load())
}
// an oversized or failing answer is no set
big := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(strings.Repeat("x", maxJWKSBody+10)))
}))
defer big.Close()
hc, _ = NewHTTPControl(HTTPControlConfig{BaseURL: big.URL, NodeID: "t1", KID: "k1", Key: priv, Client: big.Client()})
if _, err := hc.FetchJWKS(context.Background()); err == nil {
t.Fatal("an oversized document was read")
}
}
func TestJWKS_ARotationMidConnectionWorksAndAStaleSetFailsClosedForNewSessions(t *testing.T) {
clk := newFakeClock()
e, cp := httpNode(t, clk, nil)
d := e.newDevice()
conn := e.connectOK(d)
// The control plane rotates: a new key signs the sessions from now on, the old one stays published for a while.
newPub, newPriv, _ := ed25519.GenerateKey(rand.Reader)
const newKID = "cp-test-2"
cp.mu.Lock()
cp.jwks[newKID] = newPub
cp.mu.Unlock()
clk.Advance(2 * time.Second) // past the cache's minimum gap between two forced reloads
rotated := func(dev *testDevice) string {
return e.sessionSignedBy(newPriv, dev, func(h, c map[string]any) { h["kid"] = newKID })
}
// the connection that was up before the rotation renews with a session of the new key
conn.session = rotated(d)
conn.send("RENEW " + conn.session)
line := conn.recv()
m, err := parseLine(line, dirN2D)
if err != nil || m.str("verb") != "CHALLENGE" {
t.Fatalf("a session of the rotated-in key was refused at RENEW: %q", line)
}
conn.send("PROOF " + e.proof(d, conn.session, m.str("nonce"), nil))
if line = conn.recv(); !strings.HasPrefix(line, "OK ") {
t.Fatalf("renew after a rotation: %q", line)
}
// and so does a new device
d2 := e.newDevice()
conn2 := e.connect(d2)
if line := conn2.hello(func() string { return rotated(d2) }, func(s, n string) string { return e.proof(d2, s, n, nil) }); !strings.HasPrefix(line, "OK ") {
t.Fatalf("hello after a rotation: %q", line)
}
// The control plane goes away. For 29 minutes the node still vouches for the set it has; at 31 it vouches for nothing.
cp.setDown(503)
clk.Advance(29 * time.Minute)
d3 := e.newDevice()
e.cp.set(d3.id, &devicePlan{host: d3.host, err: &ControlError{Status: 503, Code: "end_to_end_unavailable"}})
if line := e.connect(d3).hello(func() string { return rotated(d3) }, func(s, n string) string { return e.proof(d3, s, n, nil) }); line != "ERR control_unavailable" {
t.Fatalf("29 minutes of silence, authorize down: %q", line) // the keys still verify; it is authorize that is unavailable
}
if strings.Contains(e.logs.String(), `"msg":"session_refused"`) {
t.Fatal("the session was refused at verification: the set should still have been trusted at 29 minutes")
}
clk.Advance(2 * time.Minute)
if line := e.connect(d3).hello(func() string { return rotated(d3) }, func(s, n string) string { return e.proof(d3, s, n, nil) }); line != "ERR control_unavailable" {
t.Fatalf("31 minutes of silence: %q", line)
}
if !strings.Contains(e.logs.String(), `"msg":"session_refused"`) || !strings.Contains(e.logs.String(), `"reason":"unknown_kid"`) {
t.Fatal("a session was verified against a set older than the policy's window")
}
// told to retry later, not told its credentials are wrong
if !strings.Contains(e.logs.String(), `"answer":"control_unavailable"`) {
t.Fatal("the refusal because of stale keys is not logged as such")
}
// the connections that were already authenticated are untouched
if !e.srv.Online(d.host) || !e.srv.Online(d2.host) {
t.Fatal("a stale JWKS closed connections that were already authenticated")
}
// and a RENEW on them is refused as unavailable too, keeping the connection
conn.session = rotated(d)
conn.send("RENEW " + conn.session)
if line := conn.recv(); line != "ERR control_unavailable" {
t.Fatalf("renew on a stale set: %q", line)
}
// back to normal
cp.setDown(0)
e.cp.set(d3.id, &devicePlan{host: d3.host, validFor: time.Hour})
clk.Advance(6 * time.Second)
if line := e.connect(d3).hello(func() string { return rotated(d3) }, func(s, n string) string { return e.proof(d3, s, n, nil) }); !strings.HasPrefix(line, "OK ") {
t.Fatalf("after the control plane came back: %q", line)
}
}
func TestJWKS_AFloodOfHellosWithRandomKidsCostsTheControlPlaneAtMostOneFetchAndHarmsNoLegitimateDevice(t *testing.T) {
clk := newFakeClock()
e, cp := httpNode(t, clk, nil)
d := e.newDevice()
_ = e.connectOK(d) // the cache has its keys
before := cp.callCount(jwksPath)
var wg sync.WaitGroup
var wrong atomic.Int64
for i := 0; i < 300; i++ {
wg.Add(1)
go func() {
defer wg.Done()
attacker := &testDevice{id: randomHex(16), key: d.key}
c, err := e.dialNode()
if err != nil {
return // the node was full: also fine
}
defer c.Close()
session := e.session(attacker, func(h, claims map[string]any) { h["kid"] = "rogue-" + randomHex(6) })
_, _ = c.Write([]byte("HELLO 2 " + session + "\n"))
_ = c.SetReadDeadline(time.Now().Add(5 * time.Second))
line, _ := bufio.NewReader(c).ReadString('\n')
if strings.TrimSpace(line) != "ERR auth" {
wrong.Add(1)
}
}()
}
wg.Wait()
if wrong.Load() != 0 {
t.Fatalf("%d of the 300 sessions with a made-up kid were not refused with auth", wrong.Load())
}
if n := cp.callCount(jwksPath) - before; n > 1 {
t.Fatalf("300 sessions with random kids made the node fetch the keys %d times", n)
}
// a legitimate device is served as before
d2 := e.newDevice()
_ = e.connectOK(d2)
}
files/tunnel/limits.go 338 lines · 12.0 KB
package main
import (
"hash/maphash"
"net/netip"
"sync"
"time"
)
// Limits of the public listener (the design plan, section 5): who may hold how many connections, how fast new ones may arrive, and who is dropped
// for trying names that are not a device. Everything is kept only in memory, for the life of the process: a source's address is a map key here and is never
// part of a log line, a metric label or a file (a log line that must tell two sources apart uses Server.short, the per-boot salted short hash).
//
// A source is an IPv4 address, or the /64 (policy node_limits.ipv6_prefix_bits) of an IPv6 one, so that rotating inside a prefix an attacker owns does not
// make new sources. An address of the OpenAI prefix list gets limits openai_per_ip_factor times larger (one egress address serves many users), is never
// banned, and is still bounded: the list is a quota class, not an allowlist (anyone can make ChatGPT call a name through those addresses).
//
// A device is a source too, and a busy one: for every client stream it serves it opens one data connection to the node, from one address. While an address
// holds a device's authenticated control connection its limits are device_per_ip_factor times larger, and the connections that prove who they are (the
// control connection once it authenticates, a data connection once it attaches) give back the token they took from the bucket: a device is not
// charged for serving its clients, only for what it tries before it has proven itself.
// Why a connection was refused before any byte of it was read, as the metrics name it.
const (
refuseBanned = "ip_banned"
refuseRate = "ip_rate"
refuseConcurrent = "ip_concurrent"
)
// ipEntry is what the node remembers of one source.
type ipEntry struct {
conns int // connections it holds now (pre-hello, and its client streams)
tokens float64 // new-connection bucket
refilled time.Time // when tokens were last computed
banUntil time.Time
strikes []uint64 // the distinct names (or junk hellos) tried in the current window, as per-boot hashes: never the names
strikeAt time.Time // when the current window began
junk uint64
}
// ipLimits is the table of sources. It is safe for concurrent use.
type ipLimits struct {
clk clock
concurrent int
rate float64 // new connections per second, and the burst
factor int // the multiple of the limits an address of the OpenAI list has
devFactor int // and an address that holds an authenticated control connection
v6bits int
max int
strikeMax int
window time.Duration
ban time.Duration
seed maphash.Seed
mu sync.Mutex
entries map[netip.Addr]*ipEntry
overflow ipEntry // the shared entry of every source that does not fit in the table
devices map[netip.Addr]int // the sources that hold an authenticated control connection, and how many
past map[netip.Addr]time.Time // the sources that held one and let go of it, and when: a device that reconnects is not an anonymous source (pastTTL)
pastTTL time.Duration
lastSweep time.Time
overflowed, bans uint64 // counters for the metrics
}
func newIPLimits(cfg *Config, clk clock, seed maphash.Seed) *ipLimits {
return &ipLimits{
clk: clk, concurrent: cfg.PerIPConcurrent, rate: float64(cfg.PerIPNewPerS), factor: cfg.OpenAIPerIPFactor, devFactor: cfg.DevicePerIPFactor, v6bits: cfg.IPv6PrefixBits,
max: cfg.IPTableMax, strikeMax: cfg.UnknownSNIPerIP, window: cfg.UnknownSNIWindow, ban: cfg.UnknownSNIBan, seed: seed,
entries: map[netip.Addr]*ipEntry{}, devices: map[netip.Addr]int{}, past: map[netip.Addr]time.Time{}, pastTTL: cfg.HintTTL,
}
}
// keyOf is the identity limits are kept under.
func (l *ipLimits) keyOf(a netip.Addr) netip.Addr {
a = a.WithZone("").Unmap()
if a.Is6() {
if p, err := a.Prefix(l.v6bits); err == nil {
return p.Addr()
}
}
return a
}
// entryLocked finds or makes the entry of key. A full table is swept of the idle ones first (at most once a second); when it is still full the source
// shares the overflow entry, which has the limits of a crowd (the table's size divided by 1024 times a source's), never none.
func (l *ipLimits) entryLocked(key netip.Addr, now time.Time) (*ipEntry, int) {
if e := l.entries[key]; e != nil {
return e, 1
}
if len(l.entries) >= l.max && now.Sub(l.lastSweep) >= time.Second {
l.sweepLocked(now)
}
if len(l.entries) >= l.max {
l.overflowed++
return &l.overflow, max(1, l.max/1024)
}
e := &ipEntry{tokens: l.rate, refilled: now}
l.entries[key] = e
return e, 1
}
// sweepLocked forgets the sources that hold nothing, are not banned, have a full bucket and no strikes in the window.
func (l *ipLimits) sweepLocked(now time.Time) {
l.lastSweep = now
for key, e := range l.entries {
if e.conns == 0 && !now.Before(e.banUntil) && now.Sub(e.refilled) >= time.Second && now.Sub(e.strikeAt) >= l.window {
delete(l.entries, key)
}
}
}
// Sweep is the periodic cleanup (a background task calls it), so an idle table shrinks without a new source having to arrive.
func (l *ipLimits) Sweep() {
l.mu.Lock()
defer l.mu.Unlock()
now := l.clk.Now()
l.sweepLocked(now)
for key, at := range l.past {
if now.Sub(at) >= l.pastTTL {
delete(l.past, key)
}
}
}
// multLocked is the multiple of a plain source's limits that key has: the crowd's (scale) for the overflow entry, times the larger of the OpenAI list's
// and a device's.
func (l *ipLimits) multLocked(key netip.Addr, scale int, openai bool) int {
mult := scale
switch {
case openai:
mult *= l.factor
case l.devices[key] > 0:
mult *= l.devFactor
}
return mult
}
// markDevice records that a device holds an authenticated control connection from addr, until the function it returns is called (once).
func (l *ipLimits) markDevice(addr netip.Addr) (unmark func()) {
key := l.keyOf(addr)
l.mu.Lock()
if l.devices[key]++; l.devices[key] == 1 {
// The bucket of this address was made when the control connection was admitted, with a plain source's burst: it now holds the device's.
e, scale := l.entries[key], 1
if e == nil {
e, scale = &l.overflow, max(1, l.max/1024)
}
e.tokens = max(e.tokens, l.rate*float64(l.multLocked(key, scale, false)))
}
l.mu.Unlock()
var once sync.Once
return func() {
once.Do(func() {
l.mu.Lock()
defer l.mu.Unlock()
if l.devices[key]--; l.devices[key] <= 0 {
delete(l.devices, key)
if len(l.past) < l.max { // a table that is full forgets the newest, never grows: the address is simply not remembered as a device's
l.past[key] = l.clk.Now()
}
}
})
}
}
// knownDevice reports whether addr holds an authenticated control connection now or let go of one less than hint_ttl_s ago: a device that reconnects after its
// connection dropped is not an anonymous source (it is unauthenticated until its PROOF, and it is what an attack on the unauthenticated budget must not lock out).
func (l *ipLimits) knownDevice(addr netip.Addr) bool {
key := l.keyOf(addr)
l.mu.Lock()
defer l.mu.Unlock()
if l.devices[key] > 0 {
return true
}
at, ok := l.past[key]
return ok && l.clk.Now().Sub(at) < l.pastTTL
}
// isDevice reports whether addr holds an authenticated control connection now.
func (l *ipLimits) isDevice(addr netip.Addr) bool {
key := l.keyOf(addr)
l.mu.Lock()
defer l.mu.Unlock()
return l.devices[key] > 0
}
// refund gives back the new-connection token that admit took for a connection that has since proved who it is (a device's control connection after its
// proof, a data connection after its ATTACH): those are not what the per-source rate is there to bound.
func (l *ipLimits) refund(addr netip.Addr, openai bool) {
key := l.keyOf(addr)
l.mu.Lock()
defer l.mu.Unlock()
e, scale := l.entries[key], 1
if e == nil {
e, scale = &l.overflow, max(1, l.max/1024)
}
e.tokens = min(l.rate*float64(l.multLocked(key, scale, openai)), e.tokens+1)
}
// admit decides whether a new connection from addr may be served. It returns the function that gives its slot back (once), or the reason it was
// refused. The new-connection bucket is charged for every accepted connection, the concurrent count until release is called.
func (l *ipLimits) admit(addr netip.Addr, openai bool) (release func(), refused string) {
now := l.clk.Now()
key := l.keyOf(addr)
l.mu.Lock()
defer l.mu.Unlock()
e, scale := l.entryLocked(key, now)
mult := l.multLocked(key, scale, openai)
if !openai && now.Before(e.banUntil) {
return nil, refuseBanned
}
burst := l.rate * float64(mult) // the bucket holds a second's worth of new connections and refills at that rate
e.tokens = min(burst, e.tokens+now.Sub(e.refilled).Seconds()*burst)
e.refilled = now
if e.tokens < 1 {
return nil, refuseRate
}
if e.conns >= l.concurrent*mult {
return nil, refuseConcurrent
}
e.tokens--
e.conns++
var once sync.Once
return func() {
once.Do(func() {
l.mu.Lock()
e.conns--
l.mu.Unlock()
})
}, ""
}
// strike records that addr tried name, which is not a device that is online (name "" for a ClientHello that is no TLS or names nothing: each one is a
// strike of its own). The same name again within the window costs nothing, so ChatGPT retrying an offline device is not a scan; more than the
// budget of distinct names bans the source for the ban period, which is reported (true) once. An address of the OpenAI list is never struck, nor are the
// sources that do not fit in the table (they share one entry that must never be banned).
func (l *ipLimits) strike(addr netip.Addr, name string, openai bool) (banned bool) {
if openai {
return false
}
now := l.clk.Now()
key := l.keyOf(addr)
l.mu.Lock()
defer l.mu.Unlock()
e, _ := l.entryLocked(key, now)
if e == &l.overflow {
// The overflow entry is a crowd, not a source: a scanner inside it must not ban every other new source for the ban period. Its limits (concurrent
// and rate) are the crowd's whole defence.
return false
}
if now.Sub(e.strikeAt) >= l.window {
e.strikes, e.strikeAt = e.strikes[:0], now
}
var h uint64
if name != "" {
h = maphash.String(l.seed, name)
for _, seen := range e.strikes {
if seen == h {
return false
}
}
} else {
e.junk++
h = e.junk // a counter cannot collide with a name's hash in practice, and a repeat of it would only cost a strike
}
e.strikes = append(e.strikes, h)
if len(e.strikes) <= l.strikeMax {
return false
}
e.strikes, e.banUntil = e.strikes[:0], now.Add(l.ban)
l.bans++
return true
}
// snapshot is what the metrics report: the size of the table, the sources banned now, the bans and the overflows so far.
func (l *ipLimits) snapshot() (size, banned int, bans, overflowed uint64) {
now := l.clk.Now()
l.mu.Lock()
defer l.mu.Unlock()
for _, e := range l.entries {
if now.Before(e.banUntil) {
banned++
}
}
return len(l.entries), banned, l.bans, l.overflowed
}
// bucket is a token bucket: rate tokens a second up to burst.
type bucket struct {
tokens float64
refilled time.Time
}
// take spends one token at now; it reports false when there is none.
func (b *bucket) take(now time.Time, rate, burst float64) bool {
if b.refilled.IsZero() {
b.tokens = burst
} else {
b.tokens = min(burst, b.tokens+now.Sub(b.refilled).Seconds()*rate)
}
b.refilled = now
if b.tokens < 1 {
return false
}
b.tokens--
return true
}
// logSampler lets a log line that a flood can repeat out at most once a second per reason, with how many were left out: an attack must not fill the disk
// through the log (nor hide the other lines in it).
type logSampler struct {
clk clock
mu sync.Mutex
at map[string]time.Time
cut map[string]int
}
func newLogSampler(clk clock) *logSampler {
return &logSampler{clk: clk, at: map[string]time.Time{}, cut: map[string]int{}}
}
// allow reports whether a line for reason may be written now, and how many of its kind were left out since the last one.
func (l *logSampler) allow(reason string) (ok bool, suppressed int) {
now := l.clk.Now()
l.mu.Lock()
defer l.mu.Unlock()
if last, seen := l.at[reason]; seen && now.Sub(last) < time.Second {
l.cut[reason]++
return false, 0
}
l.at[reason] = now
suppressed = l.cut[reason]
delete(l.cut, reason)
return true, suppressed
}
files/tunnel/limits_test.go 991 lines · 35.9 KB
package main
import (
"context"
"crypto/tls"
"encoding/json"
"fmt"
"hash/maphash"
"io"
"net"
"net/netip"
"reflect"
"strings"
"sync"
"syscall"
"testing"
"time"
)
// The limits of the public listener (limits.go): per source, per device host, global, and the ban for trying names that are not a device.
func newTestLimits(t *testing.T, mutate func(*Config)) (*ipLimits, *fakeClock) {
t.Helper()
cfg := DefaultConfig()
if mutate != nil {
mutate(&cfg)
}
fc := newFakeClock()
return newIPLimits(&cfg, newClock(fc.Now, fc.AfterFunc), maphash.MakeSeed()), fc
}
func TestDefaultConfigHardeningIsThePolicysNumbers(t *testing.T) {
cfg := DefaultConfig()
for name, c := range map[string]struct{ got, key int }{
"PerIPConcurrent": {cfg.PerIPConcurrent, ctPolInt("node_limits.per_ip_concurrent")},
"PerIPNewPerS": {cfg.PerIPNewPerS, ctPolInt("node_limits.per_ip_new_per_s")},
"IPv6PrefixBits": {cfg.IPv6PrefixBits, ctPolInt("node_limits.ipv6_prefix_bits")},
"IPTableMax": {cfg.IPTableMax, ctPolInt("node_limits.ip_table_max")},
"OpenAIPerIPFactor": {cfg.OpenAIPerIPFactor, ctPolInt("node_limits.openai_per_ip_factor")},
"DevicePerIPFactor": {cfg.DevicePerIPFactor, ctPolInt("node_limits.device_per_ip_factor")},
"UnknownSNIPerIP": {cfg.UnknownSNIPerIP, ctPolInt("node_limits.unknown_sni_per_ip")},
"LabelAnonSlice": {cfg.LabelAnonSlice, ctPolInt("node_limits.label_anon_slice")},
"LabelNewPerS": {cfg.LabelNewPerS, ctPolInt("node_limits.label_new_per_s")},
"LabelNewBurst": {cfg.LabelNewBurst, ctPolInt("node_limits.label_new_burst")},
"OpenAIReservedPct": {cfg.OpenAIReservedPct, ctPolInt("node_limits.openai_reserved_pct")},
"HintsPerLabel": {cfg.HintsPerLabel, ctPolInt("node_limits.hint_addresses_per_label")},
"GatedStreamsPerDay": {cfg.GatedStreamsPerDay, ctPolInt("gated.streams_per_day")},
} {
if c.got != c.key {
t.Errorf("%s = %d, policy %d", name, c.got, c.key)
}
}
for name, c := range map[string]struct {
got time.Duration
key string
}{
"UnknownSNIWindow": {cfg.UnknownSNIWindow, "node_limits.unknown_sni_window_s"},
"UnknownSNIBan": {cfg.UnknownSNIBan, "node_limits.unknown_sni_ban_s"},
"HintTTL": {cfg.HintTTL, "node_limits.hint_ttl_s"},
"TCPKeepalive": {cfg.TCPKeepalive, "node_limits.tcp_keepalive_s"},
"OpenAIPrefixRefresh": {cfg.OpenAIPrefixRefresh, "node_limits.openai_prefix_refresh_s"},
"HelloTimeout": {cfg.HelloTimeout, "node_limits.client_hello_deadline_s"},
"StreamIdle": {cfg.StreamIdle, "timing.stream_idle_s"},
"StreamMaxAge": {cfg.StreamMaxAge, "timing.stream_max_age_s"},
} {
if c.got != time.Duration(ctPolInt(c.key))*time.Second {
t.Errorf("%s = %s, policy %s = %d", name, c.got, c.key, ctPolInt(c.key))
}
}
if int64(cfg.GatedBytesPerDay) != int64(ctPol("gated.bytes_per_day").(float64)) {
t.Errorf("GatedBytesPerDay = %d", cfg.GatedBytesPerDay)
}
if cfg.LabelAnonSlice+ctPolInt("node_limits.label_hint_slice") != cfg.MaxStreamsPerHost {
t.Errorf("the anonymous slice %d and the hint slice %d are not the whole host cap %d", cfg.LabelAnonSlice, ctPolInt("node_limits.label_hint_slice"), cfg.MaxStreamsPerHost)
}
}
func TestConfig_HardeningIsValidated(t *testing.T) {
for name, mutate := range map[string]func(*Config){
"no per-source cap": func(c *Config) { c.PerIPConcurrent = 0 },
"no per-source rate": func(c *Config) { c.PerIPNewPerS = 0 },
"a prefix of 0 bits": func(c *Config) { c.IPv6PrefixBits = 0 },
"a prefix of 129 bits": func(c *Config) { c.IPv6PrefixBits = 129 },
"the whole node reserved": func(c *Config) { c.OpenAIReservedPct = 100 },
"no ban": func(c *Config) { c.UnknownSNIBan = 0 },
"no hint life": func(c *Config) { c.HintTTL = 0 },
"no gated budget": func(c *Config) { c.GatedBytesPerDay = 0 },
"no keepalive": func(c *Config) { c.TCPKeepalive = 0 },
"no device factor": func(c *Config) { c.DevicePerIPFactor = 0 },
"metrics on the network": func(c *Config) { c.MetricsAddr = "0.0.0.0:9101" },
"metrics on a host name": func(c *Config) { c.MetricsAddr = "example.org:9101" },
"metrics without a port": func(c *Config) { c.MetricsAddr = "127.0.0.1" },
"prefix list over http": func(c *Config) { c.OpenAIPrefixesURL = "http://example.org/list.json" },
"prefix list with a secret": func(c *Config) { c.OpenAIPrefixesURL = "https://user:pass" + "@example.org/list.json" },
} {
cfg := DefaultConfig()
cfg.HTTPSAddr, cfg.HTTPAddr = "", ""
cfg.NodeID = "t1"
cfg.Keys, cfg.Control = StaticKeys{}, &fakeControl{}
mutate(&cfg)
if err := cfg.validate(); err == nil {
t.Errorf("%s: accepted", name)
}
}
good := DefaultConfig()
good.HTTPSAddr, good.HTTPAddr = "", ""
good.NodeID = "t1"
good.Keys, good.Control = StaticKeys{}, &fakeControl{}
good.MetricsAddr, good.OpenAIPrefixesURL = "[::1]:9101", "http://127.0.0.1:8080/list.json"
if err := good.validate(); err != nil {
t.Errorf("a loopback metrics address and a loopback mirror: %v", err)
}
}
func TestIPLimits_ConcurrentCapAndRateBucketPerSource(t *testing.T) {
l, fc := newTestLimits(t, func(c *Config) { c.PerIPConcurrent, c.PerIPNewPerS = 3, 5 })
a := netip.MustParseAddr("198.51.100.7")
var releases []func()
for i := 0; i < 3; i++ {
rel, why := l.admit(a, false)
if why != "" {
t.Fatalf("connection %d: %s", i, why)
}
releases = append(releases, rel)
}
if _, why := l.admit(a, false); why != refuseConcurrent {
t.Fatalf("a fourth concurrent connection: %q", why)
}
// another source is untouched
if _, why := l.admit(netip.MustParseAddr("198.51.100.8"), false); why != "" {
t.Fatalf("another source: %q", why)
}
releases[0]()
releases[0]() // a second release of the same slot gives nothing back
if _, why := l.admit(a, false); why != "" {
t.Fatalf("after one release: %q", why)
}
if _, why := l.admit(a, false); why != refuseConcurrent {
t.Fatalf("the double release freed a slot that was not its own: %q", why)
}
// the bucket: 5 new connections in a second, however fast they are released
b := netip.MustParseAddr("203.0.113.1")
admitted := 0
for i := 0; i < 20; i++ {
rel, why := l.admit(b, false)
if why == "" {
admitted++
rel()
} else if why != refuseRate {
t.Fatalf("%q", why)
}
}
if admitted != 5 {
t.Fatalf("admitted %d in one instant, want the burst of 5", admitted)
}
fc.Advance(400 * time.Millisecond)
got := 0
for i := 0; i < 10; i++ {
if rel, why := l.admit(b, false); why == "" {
got++
rel()
}
}
if got != 2 {
t.Fatalf("after 400 ms %d more were admitted, want 2 (5 per second)", got)
}
}
func TestIPLimits_AnIPv6SourceIsItsPrefixAndAnOpenAIAddressHasLargerLimits(t *testing.T) {
l, _ := newTestLimits(t, func(c *Config) { c.PerIPConcurrent, c.PerIPNewPerS, c.OpenAIPerIPFactor = 2, 100, 4 })
hold := func(addr string, openai bool) string {
_, why := l.admit(netip.MustParseAddr(addr), openai)
return why
}
// ten addresses of one /64 are one source
for i := 0; i < 2; i++ {
if why := hold(fmt.Sprintf("2001:db8:aaaa:1::%x", i+1), false); why != "" {
t.Fatalf("%q", why)
}
}
for i := 3; i < 10; i++ {
if why := hold(fmt.Sprintf("2001:db8:aaaa:1::%x", i), false); why != refuseConcurrent {
t.Fatalf("address %d of the same /64: %q", i, why)
}
}
// the next /64 is another source
if why := hold("2001:db8:aaaa:2::1", false); why != "" {
t.Fatalf("the next /64: %q", why)
}
// an address of the OpenAI list holds factor times as many
got := 0
for i := 0; i < 20; i++ {
if hold("192.0.2.9", true) == "" {
got++
}
}
if got != 8 {
t.Fatalf("an OpenAI address held %d, want 2*4", got)
}
// an IPv4-mapped address is the IPv4 address
if why := hold("::ffff:192.0.2.77", false); why != "" {
t.Fatalf("%q", why)
}
for i := 0; i < 2; i++ {
_, why := l.admit(netip.MustParseAddr("192.0.2.77"), false)
if i == 0 && why != "" {
t.Fatalf("%q", why)
}
if i == 1 && why != refuseConcurrent {
t.Fatalf("the mapped and the plain address are two sources: %q", why)
}
}
}
func TestIPLimits_UnknownNamesBanASourceForAWhileAndOnlyDistinctNamesCount(t *testing.T) {
l, fc := newTestLimits(t, func(c *Config) {
c.UnknownSNIPerIP, c.UnknownSNIWindow, c.UnknownSNIBan = 3, time.Minute, 10*time.Minute
})
a := netip.MustParseAddr("198.51.100.20")
// the same offline device asked for again and again is not a scan
for i := 0; i < 50; i++ {
if l.strike(a, "offline.example.org", false) {
t.Fatal("a repeated name banned the source")
}
}
for _, name := range []string{"b", "c"} {
if l.strike(a, name+".example.org", false) {
t.Fatalf("banned at %s", name)
}
}
if !l.strike(a, "d.example.org", false) {
t.Fatal("the fourth distinct name (budget 3) did not ban")
}
if _, why := l.admit(a, false); why != refuseBanned {
t.Fatalf("a banned source: %q", why)
}
if _, why := l.admit(netip.MustParseAddr("198.51.100.21"), false); why != "" {
t.Fatalf("a ban reached another source: %q", why)
}
fc.Advance(9 * time.Minute)
if _, why := l.admit(a, false); why != refuseBanned {
t.Fatalf("still banned at 9 minutes: %q", why)
}
fc.Advance(2 * time.Minute)
if _, why := l.admit(a, false); why != "" {
t.Fatalf("the ban outlived its time: %q", why)
}
// the window: strikes older than it are forgotten
b := netip.MustParseAddr("198.51.100.30")
l.strike(b, "x1", false)
l.strike(b, "x2", false)
l.strike(b, "x3", false)
fc.Advance(2 * time.Minute)
if l.strike(b, "x4", false) {
t.Fatal("strikes older than the window counted")
}
// junk (no TLS, no name) counts every time: there is no name to be the same
c := netip.MustParseAddr("198.51.100.40")
banned := false
for i := 0; i < 4 && !banned; i++ {
banned = l.strike(c, "", false)
}
if !banned {
t.Fatal("four junk hellos did not ban")
}
// an address of the OpenAI list is never struck, whatever it tries
o := netip.MustParseAddr("192.0.2.50")
for i := 0; i < 100; i++ {
if l.strike(o, fmt.Sprintf("n%d.example.org", i), true) {
t.Fatal("an OpenAI address was banned")
}
}
if _, why := l.admit(o, true); why != "" {
t.Fatalf("%q", why)
}
}
func TestIPLimits_ATableThatIsFullSharesAnOverflowEntryAndShrinksWhenIdle(t *testing.T) {
l, fc := newTestLimits(t, func(c *Config) { c.IPTableMax, c.PerIPConcurrent, c.PerIPNewPerS = 4, 1, 1000 })
for i := 0; i < 4; i++ {
rel, why := l.admit(netip.MustParseAddr(fmt.Sprintf("198.51.100.%d", i+1)), false)
if why != "" {
t.Fatal(why)
}
rel()
}
// the table is full of idle entries: a new source sweeps them and gets its own
fc.Advance(2 * time.Second)
if _, why := l.admit(netip.MustParseAddr("198.51.100.99"), false); why != "" {
t.Fatalf("%q", why)
}
size, _, _, overflowed := l.snapshot()
if size > 4 || overflowed != 0 {
t.Fatalf("size %d, overflowed %d", size, overflowed)
}
// four sources that hold connections cannot be swept: the fifth shares the overflow entry, which is bounded as a crowd (not zero, not unbounded)
l2, _ := newTestLimits(t, func(c *Config) { c.IPTableMax, c.PerIPConcurrent, c.PerIPNewPerS = 4, 1, 1000 })
for i := 0; i < 4; i++ {
if _, why := l2.admit(netip.MustParseAddr(fmt.Sprintf("198.51.100.%d", i+1)), false); why != "" {
t.Fatal(why)
}
}
if _, why := l2.admit(netip.MustParseAddr("203.0.113.1"), false); why != "" {
t.Fatalf("the first of the overflow: %q", why)
}
if _, _, _, overflowed := l2.snapshot(); overflowed == 0 {
t.Fatal("no overflow was counted")
}
size, _, _, _ = l2.snapshot()
if size != 4 {
t.Fatalf("the table grew past its bound: %d", size)
}
// a later sweep drops what is idle
l3, fc3 := newTestLimits(t, func(c *Config) { c.IPTableMax, c.PerIPNewPerS = 100, 1000 })
for i := 0; i < 30; i++ {
rel, _ := l3.admit(netip.MustParseAddr(fmt.Sprintf("198.51.100.%d", i+1)), false)
rel()
}
fc3.Advance(3 * time.Minute)
l3.Sweep()
if size, _, _, _ := l3.snapshot(); size != 0 {
t.Fatalf("an idle table kept %d entries", size)
}
}
func TestBucketAndLogSampler(t *testing.T) {
var b bucket
now := time.Now()
taken := 0
for i := 0; i < 10; i++ {
if b.take(now, 2, 3) {
taken++
}
}
if taken != 3 {
t.Fatalf("a burst of 3 gave %d", taken)
}
if !b.take(now.Add(time.Second), 2, 3) || !b.take(now.Add(time.Second), 2, 3) || b.take(now.Add(time.Second), 2, 3) {
t.Fatal("two tokens a second")
}
fc := newFakeClock()
s := newLogSampler(newClock(fc.Now, fc.AfterFunc))
if ok, cut := s.allow("r"); !ok || cut != 0 {
t.Fatal("the first line is written")
}
for i := 0; i < 5; i++ {
if ok, _ := s.allow("r"); ok {
t.Fatal("a repeat inside a second was written")
}
}
if ok, _ := s.allow("other"); !ok {
t.Fatal("another reason has its own second")
}
fc.Advance(1100 * time.Millisecond)
if ok, cut := s.allow("r"); !ok || cut != 5 {
t.Fatalf("ok=%v, left out %d, want 5", ok, cut)
}
}
// ------------------------------------------------------------------------------------------------------------------------------------
// The node: what a source can do to it
// ------------------------------------------------------------------------------------------------------------------------------------
func TestNode_OneSourceCannotHoldMoreThanThePerIPCapNorFillTheGlobalBudget(t *testing.T) {
release := make(chan struct{})
e := newVIPEnv(t, func(e *testEnv, c *Config) {
c.PerIPConcurrent, c.MaxUnauth, c.HelloTimeout = 4, 16, 3*time.Second
})
d := e.newDevice()
dc := e.connectOK(d)
dc.serve(holdHandler(d, release))
defer close(release)
// pre-hello: the attacker opens 40 connections and says nothing. Only 4 are served, and the budget everybody shares is not touched beyond them.
var idle []net.Conn
for i := 0; i < 40; i++ {
c, err := e.vip.dial("198.51.100.66", e.httpsAddr)
if err != nil {
t.Fatal(err)
}
defer c.Close()
idle = append(idle, c)
}
eventually(t, "the node counted the refused ones", 3*time.Second, func() bool { return e.rejected(refuseConcurrent) >= 36 })
if got := e.srv.unauth.Load(); got > 4 {
t.Fatalf("one source holds %d pre-hello slots, cap 4", got)
}
// somebody else is served through it
good, err := e.clientFrom("203.0.113.5", d)
if err != nil {
t.Fatalf("a client of another source while one floods: %v", err)
}
_ = good.Close()
// streams: the same source holds at most 4 streams to the device, the next one is dropped with no answer at all
for _, c := range idle {
_ = c.Close()
}
eventually(t, "the slots of the closed connections are back", 3*time.Second, func() bool { return e.srv.unauth.Load() == 0 })
var held []*tls.Conn
for i := 0; i < 8; i++ {
tc, err := e.clientFrom("198.51.100.66", d)
if err == nil {
held = append(held, tc)
}
}
if len(held) != 4 {
t.Fatalf("one source held %d streams, cap 4", len(held))
}
// (the stream of the other source is still open too: the device holds it until the half-close timeout)
eventually(t, "the streams open are the 4 held and the other source's", 3*time.Second, func() bool { return e.metricValue(`all_tunnel_streams_open`) == 5 })
e.checkTranscript()
}
func TestNode_ANewConnectionRateLimitsASourceAndItRecoversWithTime(t *testing.T) {
e := newVIPEnv(t, func(e *testEnv, c *Config) { c.PerIPNewPerS, c.PerIPConcurrent = 5, 1000 })
dropped := 0
for i := 0; i < 15; i++ {
answer, closed := e.helloFrom("198.51.100.90", "nothing.devices.all-e2e.test", 300*time.Millisecond)
if closed && len(answer) == 0 {
dropped++
}
}
if dropped != 10 {
t.Fatalf("%d connections dropped without an answer, want 10 of 15 (burst of 5)", dropped)
}
if e.rejected(refuseRate) != 10 {
t.Fatalf("counted %d", e.rejected(refuseRate))
}
e.offsetNS.Add(int64(3 * time.Second))
if answer, _ := e.helloFrom("198.51.100.90", "nothing.devices.all-e2e.test", 300*time.Millisecond); len(answer) == 0 {
t.Fatal("still dropped after the bucket refilled")
}
}
func TestNode_UnknownNamesBanTheSourceAtAccept(t *testing.T) {
e := newVIPEnv(t, func(e *testEnv, c *Config) {
c.UnknownSNIPerIP, c.UnknownSNIBan, c.PerIPNewPerS = 3, 5*time.Minute, 1000
})
d := e.newDevice()
dc := e.connectOK(d)
dc.serve(nil)
scanner := "198.51.100.101"
for i := 0; i < 3; i++ {
answer, _ := e.helloFrom(scanner, fmt.Sprintf("%020d.devices.all-e2e.test", i), time.Second)
if len(answer) != 7 { // a TLS alert: the node answered
t.Fatalf("name %d: %v", i, answer)
}
}
// the fourth different name goes over the budget: it is still answered (the alert), and from then on the source is dropped at accept
if answer, _ := e.helloFrom(scanner, "zzzzzzzzzzzzzzzzzzzz.devices.all-e2e.test", time.Second); len(answer) != 7 {
t.Fatalf("the name that went over: %v", answer)
}
answer, closed := e.helloFrom(scanner, d.host, time.Second)
if !closed || len(answer) != 0 {
t.Fatalf("a banned source got %v (closed %v)", answer, closed)
}
if e.rejected(refuseBanned) != 1 || e.metricValue(`all_tunnel_ip_bans_total`) != 1 {
t.Fatalf("counters: banned %d, bans %d", e.rejected(refuseBanned), e.metricValue(`all_tunnel_ip_bans_total`))
}
// the device's clients from anywhere else are served; so is ChatGPT retrying an OFFLINE device from one address (one name, however often)
if tc, err := e.clientFrom("203.0.113.9", d); err != nil {
t.Fatalf("another source: %v", err)
} else {
_ = tc.Close()
}
for i := 0; i < 10; i++ {
if answer, _ := e.helloFrom("203.0.113.10", "offlineoffline22222.devices.all-e2e.test", time.Second); len(answer) != 7 {
t.Fatalf("retry %d of an offline name: %v", i, answer)
}
}
if answer, closed := e.helloFrom("203.0.113.10", d.host, time.Second); closed && len(answer) == 0 {
t.Fatal("a source retrying one offline name was banned")
}
// the ban ends
e.offsetNS.Add(int64(6 * time.Minute))
if answer, closed := e.helloFrom(scanner, d.host, time.Second); closed && len(answer) == 0 {
t.Fatal("still banned after the ban time")
}
}
func TestNode_ClientsThatAreNotTLSAreStruckAndAnOpenAIAddressIsNeverBanned(t *testing.T) {
var vipPrefix = "192.0.2.0/24"
e := newVIPEnv(t, func(e *testEnv, c *Config) {
c.UnknownSNIPerIP, c.PerIPNewPerS, c.PerIPConcurrent = 3, 1000, 1000
c.PrefixFetch = func(context.Context) ([]byte, error) {
return []byte(`{"prefixes":[{"ipv4Prefix":"` + vipPrefix + `"}]}`), nil
}
})
eventually(t, "the prefix list is in", 3*time.Second, func() bool { return e.srv.openai.size() == 1 })
junk := func(ip string) bool {
raw, err := e.vip.dial(ip, e.httpsAddr)
if err != nil {
t.Fatal(err)
}
defer raw.Close()
_, _ = raw.Write([]byte("GET / HTTP/1.1\r\nHost: x\r\n\r\n"))
_ = raw.SetReadDeadline(time.Now().Add(time.Second))
_, _ = io.ReadAll(raw)
return true
}
for i := 0; i < 8; i++ {
junk("198.51.100.150")
junk("192.0.2.150") // ChatGPT's address
}
if answer, closed := e.helloFrom("198.51.100.150", "x.devices.all-e2e.test", time.Second); !closed || len(answer) != 0 {
t.Fatalf("a source that sent junk four times was served: %v", answer)
}
if answer, _ := e.helloFrom("192.0.2.150", "x.devices.all-e2e.test", time.Second); len(answer) != 7 {
t.Fatalf("an OpenAI address was dropped: %v", answer)
}
}
func TestNode_SlowlorisBeforeTheClientHelloIsDroppedAtTheDeadlineWithoutAnOpen(t *testing.T) {
// The real policy deadline (5 s), not a shortened one.
e := newTestEnv(t, func(c *Config) { c.HelloTimeout = DefaultConfig().HelloTimeout })
if DefaultConfig().HelloTimeout != 5*time.Second {
t.Fatalf("the policy deadline is %s", DefaultConfig().HelloTimeout)
}
d := e.newDevice()
dc := e.connectOK(d)
ds := dc.serve(nil)
ds.noAttach = true
hello := helloOf(d.host, 0, 16000)
c, err := net.DialTimeout("tcp", e.httpsAddr, 3*time.Second)
if err != nil {
t.Fatal(err)
}
defer c.Close()
started := time.Now()
done := make(chan time.Duration, 1)
go func() {
_ = c.SetReadDeadline(time.Now().Add(15 * time.Second))
_, _ = io.ReadAll(c)
done <- time.Since(started)
}()
// one byte a second, never the whole hello
for i := 0; i < len(hello)/2 && time.Since(started) < 8*time.Second; i++ {
if _, err := c.Write(hello[i : i+1]); err != nil {
break
}
time.Sleep(time.Second)
}
select {
case took := <-done:
if took < 4500*time.Millisecond || took > 6500*time.Millisecond {
t.Fatalf("dropped after %s, want about 5 s", took)
}
case <-time.After(10 * time.Second):
t.Fatal("the slow client was never dropped")
}
if ds.openCount() != 0 {
t.Fatal("an OPEN was sent for a ClientHello that never completed")
}
if e.logs.Count(`"reason":"hello_timeout"`) < 1 {
t.Fatalf("logs:\n%s", e.logs.String())
}
if e.rejected("hello_timeout") != 1 {
t.Fatalf("counted %d", e.rejected("hello_timeout"))
}
}
func TestNode_ManyHeldStreamsFromManySourcesLeaveRoomForASourceTheDeviceMarkedGood(t *testing.T) {
release := make(chan struct{})
e := newVIPEnv(t, func(e *testEnv, c *Config) { c.PerIPNewPerS = 1000 })
d := e.newDevice()
dc := e.connectOK(d)
ds := dc.serve(holdHandler(d, release))
defer close(release)
// 20 attacker sources, 4 attempts each: 80 attempts to hold 64 streams. The anonymous slice (16) is all they get.
held := 0
for i := 0; i < 20; i++ {
for j := 0; j < 4; j++ {
if _, err := e.clientFrom(fmt.Sprintf("198.51.100.%d", 100+i), d); err == nil {
held++
}
}
}
if held != DefaultConfig().LabelAnonSlice {
t.Fatalf("the attackers held %d streams, want the anonymous slice of %d", held, DefaultConfig().LabelAnonSlice)
}
if e.logs.Count(`"reason":"host_limit"`) < 1 {
t.Fatalf("no host_limit in the logs:\n%s", e.logs.String())
}
// ChatGPT's address has not been marked: it cannot get in either, which is the price of a slice that is anonymous
if _, err := e.clientFrom("203.0.113.77", d); err == nil {
t.Fatal("an unmarked source got into a full anonymous slice")
}
// the device marks it (it just completed authenticated calls); now it has the reserved slice
dc.send("HINT 203.0.113.77 203.0.113.78 203.0.113.79 203.0.113.80")
eventually(t, "the hint is in the table", 3*time.Second, func() bool { return e.srv.hints.good(d.host, netip.MustParseAddr("203.0.113.80")) })
tc, err := e.clientFrom("203.0.113.77", d)
if err != nil {
t.Fatalf("a marked source with the anonymous slice full: %v", err)
}
defer tc.Close()
eventually(t, "the OPEN of the marked source", 3*time.Second, func() bool {
for _, o := range ds.openList() {
if o.str("peer_ip") == "203.0.113.77" {
return o.str("class") == "hint"
}
}
return false
})
// the marked sources can hold up to the whole host cap (16 held + 48 reserved), and no more
more := 1
for i := 0; i < 80; i++ {
if _, err := e.clientFrom(fmt.Sprintf("203.0.113.%d", 77+i%4), d); err == nil {
more++
}
}
if want := ctPolInt("node_limits.label_streams_max") - held; more != want {
t.Fatalf("the marked sources held %d more streams, want %d (the whole cap of the host)", more, want)
}
// the mark lasts hint_ttl_s and no longer
e.offsetNS.Add(int64(DefaultConfig().HintTTL + time.Minute))
if e.srv.hints.good(d.host, netip.MustParseAddr("203.0.113.77")) {
t.Fatal("the mark outlived its TTL")
}
}
func TestNode_AHintMarksAddressesOnlyForTheHostOfTheDeviceThatSentIt(t *testing.T) {
e := newVIPEnv(t, nil)
a, b := e.newDevice(), e.newDevice()
ca, cb := e.connectOK(a), e.connectOK(b)
ca.serve(nil)
cb.serve(nil)
ca.send("HINT 203.0.113.50 2001:db8::50")
eventually(t, "the hint", 3*time.Second, func() bool { return e.srv.hints.good(a.host, netip.MustParseAddr("203.0.113.50")) })
if e.srv.hints.good(b.host, netip.MustParseAddr("203.0.113.50")) {
t.Fatal("a device marked an address for another device's host")
}
if !e.srv.hints.good(a.host, netip.MustParseAddr("2001:db8::50")) {
t.Fatal("the IPv6 address was not marked")
}
// a HINT before the OK marks nothing: the connection has no host yet
c := e.connect(e.newDevice())
c.send("HINT 203.0.113.51")
time.Sleep(100 * time.Millisecond)
if e.srv.hints.size() != 2 {
t.Fatalf("%d addresses in the table", e.srv.hints.size())
}
}
func TestNode_PerHostNewStreamRateIsKeptForSourcesNotMarkedGood(t *testing.T) {
release := make(chan struct{})
e := newVIPEnv(t, func(e *testEnv, c *Config) { c.LabelNewPerS, c.LabelNewBurst = 2, 3 })
d := e.newDevice()
dc := e.connectOK(d)
dc.serve(holdHandler(d, release))
defer close(release)
ok := 0
for i := 0; i < 8; i++ {
if _, err := e.clientFrom(fmt.Sprintf("198.51.100.%d", 10+i), d); err == nil {
ok++
}
}
if ok != 3 {
t.Fatalf("%d streams began in an instant, want the burst of 3", ok)
}
if e.rejected("label_rate") != 5 {
t.Fatalf("counted %d", e.rejected("label_rate"))
}
// a marked source is not paced by it (the stream cap still applies)
dc.send("HINT 203.0.113.1")
eventually(t, "the hint", 3*time.Second, func() bool { return e.srv.hints.good(d.host, netip.MustParseAddr("203.0.113.1")) })
for i := 0; i < 6; i++ {
if _, err := e.clientFrom("203.0.113.1", d); err != nil {
t.Fatalf("marked stream %d: %v", i, err)
}
}
// and time refills the bucket
e.offsetNS.Add(int64(2 * time.Second))
if _, err := e.clientFrom("198.51.100.99", d); err != nil {
t.Fatalf("after a second the bucket had tokens again: %v", err)
}
}
func TestNode_TheGlobalBudgetKeepsAShareForOpenAIAndMarkedSources(t *testing.T) {
release := make(chan struct{})
e := newVIPEnv(t, func(e *testEnv, c *Config) {
c.MaxStreams, c.OpenAIReservedPct, c.LabelAnonSlice = 8, 25, 64
c.PrefixFetch = func(context.Context) ([]byte, error) {
return []byte(`{"prefixes":[{"ipv4Prefix":"192.0.2.0/24"}]}`), nil
}
})
eventually(t, "the prefix list is in", 3*time.Second, func() bool { return e.srv.openai.size() == 1 })
d := e.newDevice()
dc := e.connectOK(d)
ds := dc.serve(holdHandler(d, release))
defer close(release)
anon := 0
for i := 0; i < 12; i++ {
if _, err := e.clientFrom(fmt.Sprintf("198.51.100.%d", 10+i), d); err == nil {
anon++
}
}
if anon != 6 {
t.Fatalf("anonymous sources held %d of 8, want 6 (25%% reserved)", anon)
}
if e.logs.Count(`"reason":"anon_global_limit"`) < 1 {
t.Fatalf("logs:\n%s", e.logs.String())
}
for i := 0; i < 2; i++ {
if _, err := e.clientFrom("192.0.2.9", d); err != nil {
t.Fatalf("OpenAI stream %d: %v", i, err)
}
}
if _, err := e.clientFrom("192.0.2.10", d); err == nil {
t.Fatal("the budget of 8 was exceeded")
}
classes := map[string]int{}
for _, o := range ds.openList() {
classes[o.str("class")]++
}
if classes["openai"] != 2 || classes["anon"] != 6 {
t.Fatalf("classes in the OPENs: %v", classes)
}
}
func TestNode_EverySocketItAcceptsHasTCPKeepalive(t *testing.T) {
cfg := DefaultConfig()
cfg.TCPKeepalive = 7 * time.Second
s := &Server{cfg: cfg}
lc := s.listenConfig()
ln, err := lc.Listen(context.Background(), "tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer ln.Close()
accepted := make(chan net.Conn, 1)
go func() {
c, _ := ln.Accept()
accepted <- c
}()
c, err := net.Dial("tcp", ln.Addr().String())
if err != nil {
t.Fatal(err)
}
defer c.Close()
srv := <-accepted
defer srv.Close()
raw, err := srv.(*net.TCPConn).SyscallConn()
if err != nil {
t.Fatal(err)
}
var on int
var gerr error
_ = raw.Control(func(fd uintptr) {
on, gerr = syscall.GetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_KEEPALIVE)
})
if gerr != nil || on == 0 {
t.Fatalf("SO_KEEPALIVE = %d, %v", on, gerr)
}
if got := s.listenConfig().KeepAliveConfig; !got.Enable || got.Idle != 7*time.Second || got.Interval != 7*time.Second || got.Count != keepaliveProbes {
t.Fatalf("%+v", got)
}
}
// ------------------------------------------------------------------------------------------------------------------------------------
// The review's findings: a device's own connections, the overflow entry, the addresses of devices
// ------------------------------------------------------------------------------------------------------------------------------------
// simultaneousClients opens one stream to d from each address at the same time and returns how many were served and what failed.
func simultaneousClients(e *testEnv, d *testDevice, addrs []string) (ok int, failed []error) {
var wg sync.WaitGroup
var mu sync.Mutex
for _, ip := range addrs {
wg.Add(1)
go func(ip string) {
defer wg.Done()
_, err := e.clientFrom(ip, d)
mu.Lock()
defer mu.Unlock()
if err != nil {
failed = append(failed, err)
} else {
ok++
}
}(ip)
}
wg.Wait()
return ok, failed
}
func TestNode_ADevicesOwnConnectionsAreNotChargedToTheAnonymousBucketAtPolicyLimits(t *testing.T) {
release := make(chan struct{})
e := newVIPEnv(t, nil) // the policy's per-source numbers: 16 held and 10 new a second per anonymous source
d := e.newDevice()
dc := e.connectOK(d)
ds := dc.serve(holdHandler(d, release))
defer close(release)
// 32 sources the device marked good: only the device's own ATTACH connections (one per stream, all from its one address) are in question here
var addrs []string
var marked []string
for i := 0; i < 32; i++ {
addrs = append(addrs, fmt.Sprintf("203.0.113.%d", 1+i))
marked = append(marked, addrs[i])
}
dc.send("HINT " + strings.Join(marked[:16], " "))
dc.send("HINT " + strings.Join(marked[16:], " "))
eventually(t, "the hints", 3*time.Second, func() bool { return e.srv.hints.good(d.host, netip.MustParseAddr(addrs[31])) })
ok, failed := simultaneousClients(e, d, addrs)
if ok != 32 {
t.Fatalf("%d of 32 simultaneous streams were served (%v): the device's ATTACHes were refused as an anonymous source's connections (ip_rate %d, ip_concurrent %d)",
ok, failed, e.rejected(refuseRate), e.rejected(refuseConcurrent))
}
if e.rejected(refuseRate)+e.rejected(refuseConcurrent) != 0 {
t.Fatalf("refused at accept: ip_rate %d, ip_concurrent %d", e.rejected(refuseRate), e.rejected(refuseConcurrent))
}
if ds.openCount() != 32 {
t.Fatalf("%d OPENs", ds.openCount())
}
// and the same device keeps its larger allowance only while it holds an authenticated control connection: when it is gone the address is anonymous again
_ = dc.c.Close()
eventually(t, "the device is gone", 3*time.Second, func() bool { return e.srv.controlCount() == 0 })
eventually(t, "the address is no longer a device's", 3*time.Second, func() bool { return !e.srv.ips.isDevice(netip.MustParseAddr("127.0.0.1")) })
}
func TestIPLimits_ADeviceAddressHasLargerLimitsAndItsAuthenticatedConnectionsGiveTheirTokenBack(t *testing.T) {
l, _ := newTestLimits(t, func(c *Config) { c.PerIPConcurrent, c.PerIPNewPerS, c.DevicePerIPFactor = 100, 3, 4 })
// an anonymous source is held to 3 new connections in an instant
a := netip.MustParseAddr("198.51.100.7")
for i := 0; i < 3; i++ {
if _, why := l.admit(a, false); why != "" {
t.Fatal(why)
}
}
if _, why := l.admit(a, false); why != refuseRate {
t.Fatalf("%q", why)
}
// the address of a device has 4 times that (12), from the moment its control connection authenticates, though its bucket was made before
b := netip.MustParseAddr("198.51.100.8")
if _, why := l.admit(b, false); why != "" { // the control connection, admitted as any source's
t.Fatal(why)
}
unmark := l.markDevice(b)
if !l.isDevice(b) {
t.Fatal("not a device")
}
var rels []func()
for i := 0; i < 12; i++ {
rel, why := l.admit(b, false)
if why != "" {
t.Fatalf("connection %d of the device: %q", i+1, why)
}
rels = append(rels, rel)
}
if _, why := l.admit(b, false); why != refuseRate {
t.Fatalf("the thirteenth after the mark: %q", why)
}
// the ones that proved who they are (attached) give their token back: all of them are in the bucket again
for _, rel := range rels {
l.refund(b, false)
rel()
}
for i := 0; i < 12; i++ {
if _, why := l.admit(b, false); why != "" {
t.Fatalf("after the refunds, connection %d: %q", i+1, why)
}
}
// a refund never fills the bucket past its burst
for i := 0; i < 50; i++ {
l.refund(b, false)
}
if got := l.entries[l.keyOf(b)].tokens; got > 12 {
t.Fatalf("%v tokens in a bucket of 12", got)
}
unmark()
unmark() // once
if l.isDevice(b) {
t.Fatal("still a device after its connection ended")
}
// two devices behind one address: it stays a device until the last one is gone
u1, u2 := l.markDevice(b), l.markDevice(b)
u1()
if !l.isDevice(b) {
t.Fatal("one device left, the address is no longer one")
}
u2()
if l.isDevice(b) || len(l.devices) != 0 {
t.Fatal("the table of devices did not empty")
}
}
func TestIPLimits_TheOverflowEntryIsNeverStruckNorBanned(t *testing.T) {
l, _ := newTestLimits(t, func(c *Config) { c.IPTableMax, c.PerIPConcurrent, c.PerIPNewPerS, c.UnknownSNIPerIP = 4, 1, 1000, 3 })
for i := 0; i < 4; i++ { // four sources that hold connections fill the table and cannot be swept
if _, why := l.admit(netip.MustParseAddr(fmt.Sprintf("198.51.100.%d", i+1)), false); why != "" {
t.Fatal(why)
}
}
scanner := netip.MustParseAddr("203.0.113.9")
for i := 0; i < 10; i++ {
if l.strike(scanner, fmt.Sprintf("n%d.example.org", i), false) {
t.Fatalf("a scanner inside the overflow banned the whole crowd with name %d", i)
}
}
if _, _, bans, _ := l.snapshot(); bans != 0 {
t.Fatalf("%d bans", bans)
}
// an innocent new source (it too lands in the overflow) is served as before
if _, why := l.admit(netip.MustParseAddr("192.0.2.77"), false); why != "" {
t.Fatalf("an innocent new source was refused: %q", why)
}
// a source with an entry of its own is still struck and banned
l2, _ := newTestLimits(t, func(c *Config) { c.UnknownSNIPerIP = 3 })
banned := false
for i := 0; i < 4; i++ {
banned = l2.strike(scanner, fmt.Sprintf("n%d.example.org", i), false)
}
if !banned {
t.Fatal("a source of its own was not banned")
}
}
func TestNode_ASourceADeviceMarkedGoodIsNeverBannedForTheNamesItAsks(t *testing.T) {
e := newVIPEnv(t, func(e *testEnv, c *Config) { c.UnknownSNIPerIP, c.PerIPNewPerS = 3, 1000 })
d := e.newDevice()
dc := e.connectOK(d)
dc.serve(nil)
dc.send("HINT 203.0.113.50")
eventually(t, "the hint", 3*time.Second, func() bool { return e.srv.hints.goodAnywhere(netip.MustParseAddr("203.0.113.50")) })
// a shared egress address that the fleet knows: many devices are offline at any moment, so it asks for many names
for i := 0; i < 10; i++ {
if answer, closed := e.helloFrom("203.0.113.50", fmt.Sprintf("off%d.%s", i, testZone), 500*time.Millisecond); closed && len(answer) == 0 {
t.Fatalf("name %d: the marked address was dropped without an answer", i)
}
}
if e.metricValue(`all_tunnel_ip_bans_total`) != 0 {
t.Fatal("a marked address was banned")
}
// the same behaviour from an address nobody marked is a scan
for i := 0; i < 10; i++ {
e.helloFrom("198.51.100.50", fmt.Sprintf("off%d.%s", i, testZone), 500*time.Millisecond)
}
if e.metricValue(`all_tunnel_ip_bans_total`) != 1 {
t.Fatal("an unmarked scanner was not banned")
}
}
func TestNode_AMetricsPortThatIsTakenDoesNotKeepTheNodeFromStarting(t *testing.T) {
taken, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer taken.Close()
e := newVIPEnv(t, func(e *testEnv, c *Config) { c.MetricsAddr = taken.Addr().String() })
if e.srv.MetricsAddr() != "" {
t.Fatalf("metrics listen on %s", e.srv.MetricsAddr())
}
if e.logs.Count(`"msg":"metrics_unavailable"`) != 1 || strings.Contains(e.logs.String(), taken.Addr().String()) {
t.Fatalf("logs:\n%s", e.logs.String())
}
d := e.newDevice()
e.connectOK(d).serve(nil)
if _, err := e.clientFrom("198.51.100.5", d); err != nil {
t.Fatalf("the node does not serve: %v", err)
}
}
// The embedded policy is a PROJECTION of contracts/policy.json once a runtime's file holds only the sections it reads: a section or key the node reads
// that the projection lacks would read as zero. This holds for every json field of Policy, whatever the projection is.
func TestPolicyCopyHoldsEveryKeyTheNodeReads(t *testing.T) {
var raw map[string]any
if err := json.Unmarshal(policyJSON, &raw); err != nil {
t.Fatal(err)
}
var walk func(path string, typ reflect.Type, at any)
walk = func(path string, typ reflect.Type, at any) {
for i := 0; i < typ.NumField(); i++ {
f := typ.Field(i)
name, _, _ := strings.Cut(f.Tag.Get("json"), ",")
if name == "" || name == "-" {
continue
}
obj, _ := at.(map[string]any)
got, ok := obj[name]
if !ok {
t.Errorf("tunnel/policy.json has no %s.%s, which the node reads (add it to the node's projection in the contract lint table and sync)", path, name)
continue
}
if f.Type.Kind() == reflect.Struct {
walk(path+"."+name, f.Type, got)
}
}
}
walk("policy", reflect.TypeOf(Policy{}), raw)
}
files/tunnel/loghygiene_test.go 151 lines · 5.9 KB
package main
import (
"context"
"fmt"
"regexp"
"strings"
"testing"
"time"
)
// No IP or SNI in persistent logs (plan section 5): a full run of the node through everything it does, then a grep of what it wrote and of its metrics
// for every address, label, host name and device id the run used, and for anything shaped like one.
func TestLogs_AFullRunLeavesNoAddressNoLabelNoNameInTheLogsOrTheMetrics(t *testing.T) {
e := newVIPEnv(t, func(e *testEnv, c *Config) {
c.UnknownSNIPerIP, c.LabelNewPerS, c.LabelNewBurst, c.PerIPConcurrent = 3, 2, 3, 4
c.GatedBytesPerDay = 70_000
c.MaxStreamsPerHost = 6
c.LabelAnonSlice = 3
c.PrefixFetch = func(context.Context) ([]byte, error) {
return []byte(`{"prefixes":[{"ipv4Prefix":"192.0.2.0/24"},{"ipv6Prefix":"2001:db8:cafe::/48"}]}`), nil
}
c.PerIPNewPerS = 1 << 20
})
eventually(t, "the prefix list", 3*time.Second, func() bool { return e.srv.openai.size() == 2 })
release := make(chan struct{})
paid, gated, offline := e.newDevice(), e.newDevice(), e.newDevice()
e.plan(gated).status, e.plan(gated).plan = statusGated, "trial"
pc := e.connectOK(paid)
pc.serve(holdHandler(paid, release))
gc := e.connectOK(gated)
gc.serve(nil)
ips := []string{"198.51.100.17", "203.0.113.200", "2001:db8:1234:5678::9", "192.0.2.77", "2001:db8:cafe::1", "198.51.100.18"}
// streams from several sources, a HINT, the label rate and the host cap
pc.send("HINT 203.0.113.200 2001:db8:1234:5678::9")
eventually(t, "the hint", 3*time.Second, func() bool { return e.srv.hints.size() == 2 })
for _, ip := range ips {
for i := 0; i < 3; i++ {
_, _ = e.clientFrom(ip, paid)
}
}
// names that are nobody's, a scanner that is banned, junk
scanner := "198.51.100.99"
names := []string{offline.host}
for i := 0; i < 5; i++ {
names = append(names, fmt.Sprintf("%c%019d.devices.all-e2e.test", 'a'+i, i))
}
for _, n := range names {
e.helloFrom(scanner, n, 500*time.Millisecond)
}
e.helloFrom(scanner, paid.host, 500*time.Millisecond)
ips = append(ips, scanner)
// the gated device runs out of bytes; the paid one renews; a refused device
if c, err := e.clientFrom("198.51.100.50", gated); err == nil {
_, _ = postOver(c, gated.host, make([]byte, 300_000))
}
ips = append(ips, "198.51.100.50")
pc.drainFeed()
if r := pc.renew(); !strings.HasPrefix(r, "OK ") {
t.Fatalf("renew: %q", r)
}
stranger := e.newDeviceNoPlan()
sc := e.connect(stranger)
sc.hello(func() string { return e.session(stranger, nil) }, func(s, n string) string { return e.proof(stranger, s, n, nil) })
// a client that is not TLS, a cut hello, a client that says nothing
raw := sendRaw(t, e, []byte("GET / HTTP/1.1\r\n\r\n"), 0)
_ = raw
go e.srv.Drain(3 * time.Second)
eventually(t, "draining", 3*time.Second, func() bool { return e.srv.Draining() })
e.helloFrom("198.51.100.60", paid.host, 500*time.Millisecond)
ips = append(ips, "198.51.100.60")
close(release)
time.Sleep(300 * time.Millisecond)
logs, metrics := e.logs.String(), e.srv.renderMetrics()
// The node's own listen address (the "listening" line, written once from its configuration) is the operator's, not a client's, and the time of each
// line is not an address however it reads.
var kept []string
for _, line := range strings.Split(logs, "\n") {
if !strings.Contains(line, `"msg":"listening"`) {
kept = append(kept, regexp.MustCompile(`"time":"[^"]*",`).ReplaceAllString(line, ""))
}
}
logs = strings.Join(kept, "\n")
if len(logs) < 2000 || !strings.Contains(logs, `"msg":"stream_open"`) || !strings.Contains(logs, `"msg":"source_banned"`) || !strings.Contains(logs, `"msg":"conn_reject"`) {
t.Fatalf("the control is empty: the run did not exercise the node (%d bytes of logs)", len(logs))
}
secrets := map[string]string{
"the loopback address": "127.0.0.1", "the loopback v6": "::1",
}
for i, ip := range ips {
secrets[fmt.Sprintf("source %d", i)] = ip
}
for _, d := range []*testDevice{paid, gated, offline, stranger} {
secrets["a label"+d.label] = d.label
secrets["a host"+d.label] = d.host
secrets["a device id"+d.label] = d.id
}
for _, n := range names {
secrets["the name "+n] = n
secrets["the label of "+n] = n[:20]
}
secrets["the zone"] = testZone
secrets["the node's own name"] = testNodeHost
for what, secret := range secrets {
for where, text := range map[string]string{"logs": logs, "metrics": metrics} {
if strings.Contains(text, secret) {
t.Errorf("%s hold %s (%q)", where, what, secret)
}
}
}
for what, re := range map[string]*regexp.Regexp{
"an IPv4 address": regexp.MustCompile(`\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b`),
"an IPv6 address": regexp.MustCompile(`(?i)\b[0-9a-f]{0,4}:[0-9a-f]{0,4}:[0-9a-f:]{2,}`),
"something label-like": regexp.MustCompile(`\b[a-z2-7]{20}\b`), // on word boundaries: a 32-character hex stream id holds a run of 20 such characters now and then
"a host name": regexp.MustCompile(`[a-z0-9-]+\.[a-z0-9-]+\.(test|org|com|net)`),
} {
for where, text := range map[string]string{"logs": logs, "metrics": metrics} {
if m := re.FindString(text); m != "" {
t.Errorf("%s hold %s: %q", where, what, m)
}
}
}
// the one way a source appears is the per-boot salted short hash, and it is not the address
re := regexp.MustCompile(`"source":"([0-9a-f]{8})"`)
m := re.FindStringSubmatch(logs)
if m == nil {
t.Fatalf("a ban was logged without the short hash:\n%s", grepLines(logs, "source_banned"))
}
if m[1] != e.srv.short(scanner) {
t.Fatalf("the hash %s is not the scanner's %s", m[1], e.srv.short(scanner))
}
// the salt is per boot: another node hashes the same address to another word
other := newVIPEnv(t, nil)
if other.srv.short(scanner) == e.srv.short(scanner) {
t.Fatal("two boots hash an address to the same word")
}
}
func grepLines(text, sub string) string {
var out []string
for _, line := range strings.Split(text, "\n") {
if strings.Contains(line, sub) {
out = append(out, line)
}
}
return strings.Join(out, "\n")
}
files/tunnel/main.go 213 lines · 9.6 KB
package main
import (
"errors"
"flag"
"fmt"
"io"
"log/slog"
"os"
"os/signal"
"strconv"
"syscall"
"time"
)
// version is overridden at build time: -ldflags "-X main.version=..."
var version = "dev"
func main() { os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) }
func envStr(key, def string) string {
if v, ok := os.LookupEnv(key); ok && v != "" {
return v
}
return def
}
func envInt(key string, def int64) int64 {
if v := os.Getenv(key); v != "" {
if n, err := strconv.ParseInt(v, 10, 64); err == nil {
return n
}
}
return def
}
func envDur(key string, def time.Duration) time.Duration {
if v := os.Getenv(key); v != "" {
if d, err := time.ParseDuration(v); err == nil {
return d
}
}
return def
}
// backend is what the node asks of the outside world: the control plane's keys, the authorize call and the feed. The production build registers the real
// ones (backend_prod.go: the HTTP client with the node's assertion, the JWKS cache); the test-node build (`-tags testnode`) registers file-driven fakes. A
// build with neither refuses to start rather than run without verifying anyone.
type backend struct {
// flags registers the backend's own flags and returns what completes cfg once the flags are parsed.
flags func(fs *flag.FlagSet, stdout io.Writer) func(cfg *Config) error
}
var theBackend *backend
// Where the production build looks by default: its /metrics and /healthz (loopback), and the list of OpenAI connector prefixes. The test node leaves both
// empty (many test nodes run side by side and none may call the network).
var (
defaultMetricsAddr string
defaultOpenAIPrefixesURL string
)
var errNoBackend = errors.New("this build has no control-plane client: use the production build, or the test-node build (-tags testnode) with its fakes")
// errDone is what a backend's complete returns when it did what the command line asked (printing a key) and the node must not start.
var errDone = errors.New("done")
func run(args []string, stdout, stderr io.Writer) int {
cfg := DefaultConfig()
fs := flag.NewFlagSet("all-tunnel", flag.ContinueOnError)
fs.SetOutput(stderr)
fs.StringVar(&cfg.HTTPSAddr, "https-addr", envStr("TUNNEL_HTTPS_ADDR", cfg.HTTPSAddr), "the public listener: devices' control and data connections and clients' TLS, told apart by SNI [TUNNEL_HTTPS_ADDR]")
fs.StringVar(&cfg.HTTPAddr, "http-addr", envStr("TUNNEL_HTTP_ADDR", cfg.HTTPAddr), "listener that serves only the node's own ACME webroot, '' disables [TUNNEL_HTTP_ADDR]")
fs.StringVar(&cfg.NodeID, "node-id", envStr("TUNNEL_NODE_ID", ""), "this node's id in the control plane, 1-16 characters of [a-z0-9] [TUNNEL_NODE_ID]")
fs.StringVar(&cfg.NodeHost, "node-host", envStr("TUNNEL_NODE_HOST", ""), "this node's own public name: the SNI that reaches the control handler [TUNNEL_NODE_HOST]")
fs.IntVar(&cfg.PublicPort, "public-port", int(envInt("TUNNEL_PUBLIC_PORT", int64(cfg.PublicPort))), "the port devices dial, only for the proof's URL [TUNNEL_PUBLIC_PORT]")
fs.StringVar(&cfg.ACMEWebroot, "acme-webroot", envStr("TUNNEL_ACME_WEBROOT", ""), "directory with .well-known/acme-challenge for the node's own certificate [TUNNEL_ACME_WEBROOT]")
certFile := fs.String("cert-file", envStr("TUNNEL_CERT_FILE", ""), "the node's public-CA certificate chain (PEM), re-read when it changes [TUNNEL_CERT_FILE]")
keyFile := fs.String("key-file", envStr("TUNNEL_KEY_FILE", ""), "the node certificate's private key (PEM) [TUNNEL_KEY_FILE]")
skewS := fs.Int64("clock-skew", envInt("TUNNEL_CLOCK_SKEW_S", int64(cfg.ClockSkew/time.Second)), "seconds a device's clock may be ahead of this node's (at most the policy's maximum) [TUNNEL_CLOCK_SKEW_S]")
fs.DurationVar(&cfg.FeedPoll, "feed-poll", envDur("TUNNEL_FEED_POLL", cfg.FeedPoll), "how often the node looks at the control plane's feed (at most the policy's) [TUNNEL_FEED_POLL]")
fs.DurationVar(&cfg.Grace, "grace", envDur("TUNNEL_GRACE", cfg.Grace), "how long a paid connection outlives its valid_until while the control plane does not answer (at most the policy's) [TUNNEL_GRACE]")
fs.DurationVar(&cfg.DrainQuiet, "drain-quiet", envDur("TUNNEL_DRAIN_QUIET", cfg.DrainQuiet), "on SIGTERM keep the open streams this long at most, then cut them and exit; the unit's TimeoutStopSec must be longer [TUNNEL_DRAIN_QUIET]")
fs.StringVar(&cfg.MetricsAddr, "metrics-addr", envStr("TUNNEL_METRICS_ADDR", defaultMetricsAddr), "loopback address of /metrics and /healthz (aggregate counters only), '' disables [TUNNEL_METRICS_ADDR]")
fs.StringVar(&cfg.OpenAIPrefixesURL, "openai-prefixes-url", envStr("TUNNEL_OPENAI_PREFIXES_URL", defaultOpenAIPrefixesURL), "where OpenAI publishes its connector prefixes (the node's openai quota class), '' disables the refresh [TUNNEL_OPENAI_PREFIXES_URL]")
fs.StringVar(&cfg.OpenAIPrefixesCache, "openai-prefixes-cache", envStr("TUNNEL_OPENAI_PREFIXES_CACHE", ""), "file that keeps the last good prefix list across restarts (public data) [TUNNEL_OPENAI_PREFIXES_CACHE]")
fs.IntVar(&cfg.MaxStreams, "max-streams", int(envInt("TUNNEL_MAX_STREAMS", int64(cfg.MaxStreams))), "max concurrent client connections [TUNNEL_MAX_STREAMS]")
fs.IntVar(&cfg.MaxStreamsPerHost, "max-streams-per-host", int(envInt("TUNNEL_MAX_STREAMS_PER_HOST", int64(cfg.MaxStreamsPerHost))), "max concurrent client connections per device [TUNNEL_MAX_STREAMS_PER_HOST]")
fs.Int64Var(&cfg.MaxStreamBytes, "max-stream-bytes", envInt("TUNNEL_MAX_STREAM_BYTES", cfg.MaxStreamBytes), "max bytes (both directions) per stream [TUNNEL_MAX_STREAM_BYTES]")
fs.DurationVar(&cfg.StreamIdle, "stream-idle", envDur("TUNNEL_STREAM_IDLE", cfg.StreamIdle), "close a stream after this long without bytes [TUNNEL_STREAM_IDLE]")
fs.DurationVar(&cfg.StreamMaxAge, "stream-max-age", envDur("TUNNEL_STREAM_MAX_AGE", cfg.StreamMaxAge), "hard cap on one stream's lifetime [TUNNEL_STREAM_MAX_AGE]")
fs.DurationVar(&cfg.HelloTimeout, "hello-timeout", envDur("TUNNEL_HELLO_TIMEOUT", cfg.HelloTimeout), "client must send its ClientHello within this [TUNNEL_HELLO_TIMEOUT]")
fs.DurationVar(&cfg.AttachTimeout, "attach-timeout", envDur("TUNNEL_ATTACH_TIMEOUT", cfg.AttachTimeout), "device must ATTACH within this after OPEN [TUNNEL_ATTACH_TIMEOUT]")
fs.DurationVar(&cfg.DeadPeer, "dead-peer", envDur("TUNNEL_DEAD_PEER", cfg.DeadPeer), "drop a control connection silent for this long [TUNNEL_DEAD_PEER]")
logFormat := fs.String("log-format", envStr("TUNNEL_LOG_FORMAT", "json"), "json or text [TUNNEL_LOG_FORMAT]")
logLevel := fs.String("log-level", envStr("TUNNEL_LOG_LEVEL", "info"), "debug, info, warn or error [TUNNEL_LOG_LEVEL]")
showVersion := fs.Bool("version", false, "print the version and exit")
var complete func(*Config) error
if theBackend != nil {
complete = theBackend.flags(fs, stdout)
}
if err := fs.Parse(args); err != nil {
if err == flag.ErrHelp {
return 0
}
return 2
}
if *showVersion {
fmt.Fprintln(stdout, "all-tunnel", version)
return 0
}
var lvl slog.Level
if err := lvl.UnmarshalText([]byte(*logLevel)); err != nil {
fmt.Fprintln(stderr, "error: bad -log-level")
return 2
}
hopts := &slog.HandlerOptions{Level: lvl}
if *logFormat == "text" {
cfg.Logger = slog.New(slog.NewTextHandler(stderr, hopts))
} else {
cfg.Logger = slog.New(slog.NewJSONHandler(stderr, hopts))
}
cfg.ClockSkew = time.Duration(*skewS) * time.Second
if complete == nil {
fmt.Fprintln(stderr, "error:", errNoBackend)
return 1
}
var reloader *certReloader
if *certFile != "" || *keyFile != "" {
var err error
if reloader, err = newCertReloader(*certFile, *keyFile, cfg.NodeHost, newClock(nil, nil), cfg.Logger); err != nil {
fmt.Fprintln(stderr, "error:", err)
return 1
}
cfg.CertSource = reloader.get
cfg.CertNotAfter = reloader.NotAfter
cfg.Background = append(cfg.Background, reloader.Run)
}
if err := complete(&cfg); err != nil {
if errors.Is(err, errDone) {
return 0
}
fmt.Fprintln(stderr, "error:", err)
return 1
}
srv, err := NewServer(cfg)
if err != nil {
fmt.Fprintln(stderr, "error:", err)
return 1
}
if err := srv.Start(); err != nil {
fmt.Fprintln(stderr, "error:", err)
return 1
}
cfg.Logger.Info("started", "version", version, "node", cfg.NodeID)
announce(stdout, srv)
return serveUntilSignalled(srv, cfg, reloader)
}
// serveUntilSignalled is the node's life after it started. SIGTERM drains (the open streams finish, up to the quiet period, then it exits; a second
// SIGTERM stops waiting), SIGINT stops at once (a developer's Ctrl-C), SIGHUP reads the node's certificate again.
func serveUntilSignalled(srv *Server, cfg Config, reloader *certReloader) int {
sigs := make(chan os.Signal, 8)
signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
defer signal.Stop(sigs)
drained := make(chan struct{})
draining := false
for {
select {
case <-drained:
return 0
case sig := <-sigs:
switch sig {
case syscall.SIGHUP:
if reloader == nil {
continue
}
if err := reloader.Reload(); err != nil {
cfg.Logger.Warn("node_cert_reload_refused", "reason", err.Error())
} else {
cfg.Logger.Info("node_cert_reloaded", "not_after", reloader.NotAfter().UTC().Format(time.RFC3339))
}
case syscall.SIGTERM:
if !draining {
draining = true
go func() {
srv.Drain(cfg.DrainQuiet)
close(drained)
}()
continue
}
cfg.Logger.Warn("drain_cut_short")
go srv.Shutdown(0)
default:
cfg.Logger.Info("shutting_down", "grace", cfg.ShutdownGrace.String())
srv.Shutdown(cfg.ShutdownGrace)
return 0
}
}
}
}
// announce is called once the node listens; the test-node build uses it to tell a harness the ports it got. The production build says nothing.
var announce = func(io.Writer, *Server) {}
files/tunnel/metrics.go 294 lines · 13.6 KB
package main
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net"
"net/http"
"runtime"
"sync/atomic"
"time"
)
// /metrics and /healthz (the design plan, section 5): loopback only, aggregate counters only. There is NO series per device, per label, per
// address or per name, by construction: every counter is a member of a family whose words are a closed list written below, and a word that is not in
// the list is counted as "other". The number of series therefore does not depend on how many devices or clients the node serves, and nothing a client
// or a device sends can make a new one. The operator's tools read it from the machine (the monitor of WP22 reads the node certificate's not_after and the
// age of the control plane's keys from here, and the bytes per user come from the devices' own heartbeats, not from the node).
// family is a counter with one label whose values are a closed list of words.
type family struct {
name, help, label string
words []string
vals []atomic.Int64
idx map[string]int
}
func newFamily(name, help, label string, words []string) *family {
f := &family{name: name, help: help, label: label, words: append([]string(nil), words...), idx: map[string]int{}}
f.vals = make([]atomic.Int64, len(f.words))
for i, w := range f.words {
f.idx[w] = i
}
if _, ok := f.idx["other"]; !ok {
f.idx["other"] = len(f.words)
f.words = append(f.words, "other")
f.vals = make([]atomic.Int64, len(f.words))
}
return f
}
func (f *family) add(word string, n int64) {
i, ok := f.idx[word]
if !ok {
i = f.idx["other"]
}
f.vals[i].Add(n)
}
func (f *family) inc(word string) { f.add(word, 1) }
// get is the count of a word (0 for one that is not in the list).
func (f *family) get(word string) int64 {
if i, ok := f.idx[word]; ok {
return f.vals[i].Load()
}
return 0
}
func (f *family) write(b *bytes.Buffer) {
fmt.Fprintf(b, "# HELP %s %s\n# TYPE %s counter\n", f.name, f.help, f.name)
for i, w := range f.words {
fmt.Fprintf(b, "%s{%s=%q} %d\n", f.name, f.label, w, f.vals[i].Load())
}
}
// The closed lists of words.
var (
// why a connection or a stream was refused before it carried anything
rejectWords = []string{
refuseBanned, refuseRate, refuseConcurrent, "unauth_limit", "unauth_anon_limit", "not_tls", "bad_hello", "hello_too_large", "hello_timeout", "closed_before_hello",
"read_error", "no_sni", "bad_sni", "unknown_host", "device_offline", "host_limit", "label_rate", "global_limit", "anon_global_limit", "draining",
"gated_streams", "gated_bytes", "attach_timeout", "control_write_failed", "shutting_down", "other",
}
// why a stream that was served ended
streamEndWords = []string{
"closed", "idle", "max_age", "byte_cap", "conn_error", "shutdown", "half_close_timeout", "gated_bytes",
closeRevoked, closeExpired, closeReleased, closeKilled, closePaused, closeDrain, "other",
}
authorizeWords = []string{"ok", "refused", "unavailable", "other"}
feedWords = []string{feedRevoke, feedPause, feedResume, feedLicenseEnd, feedRelease, feedKill, feedKillClear, "other"}
prefixWords = []string{prefixOK, prefixFetch, prefixEmpty, prefixCorrupt, prefixTooLarge, prefixTooMany, prefixBadPrefix, "other"}
directionWords = []string{"up", "down", "other"}
)
// metrics holds the node's counters. Every member is aggregate and every label value comes from a list above or from the observer's tables.
type metrics struct {
rejected *family
streamEnd *family
bytes *family
authorize *family
feedEvents *family
prefixRefresh *family
streamsOpened atomic.Int64
gatedCut atomic.Int64 // streams cut because a gated device's byte budget ran out
hintsMarked atomic.Int64
bansTotal atomic.Int64
obsOffered *family // groups a ClientHello offered
obsVersion *family // what the ServerHello selected
obsCipher *family
obsGroup *family
obsHellos atomic.Int64 // ClientHellos observed
obsECH atomic.Int64 // of them, how many carried an ECH extension
obsRetry atomic.Int64 // HelloRetryRequests
obsUnparsed atomic.Int64 // streams whose first bytes from the device were not a ServerHello the observer follows
}
func newMetrics() *metrics {
return &metrics{
rejected: newFamily("all_tunnel_conn_rejected_total", "Connections and streams refused before they carried anything, by reason.", "reason", rejectWords),
streamEnd: newFamily("all_tunnel_stream_closed_total", "Client streams that were served and ended, by why.", "reason", streamEndWords),
bytes: newFamily("all_tunnel_bytes_total", "Bytes spliced, client to device (up) and device to client (down).", "direction", directionWords),
authorize: newFamily("all_tunnel_authorize_total", "Calls to the control plane's authorize, by outcome.", "result", authorizeWords),
feedEvents: newFamily("all_tunnel_feed_events_total", "Feed events applied, by type.", "type", feedWords),
prefixRefresh: newFamily("all_tunnel_openai_prefix_refresh_total", "Refreshes of the OpenAI prefix list, by outcome.", "result", prefixWords),
obsOffered: newFamily("all_tunnel_tls_client_offered_group_total", "ClientHellos that offered a named group (each counted once per hello).", "group", obsGroupWords),
obsVersion: newFamily("all_tunnel_tls_selected_version_total", "TLS version a device's ServerHello selected.", "version", obsVersionWords),
obsCipher: newFamily("all_tunnel_tls_selected_cipher_total", "Cipher suite a device's ServerHello selected.", "cipher", obsCipherWords),
obsGroup: newFamily("all_tunnel_tls_selected_group_total", "Key-exchange group a device's ServerHello selected (none: TLS 1.2).", "group", append(append([]string(nil), obsGroupWords...), obsNoGroup)),
}
}
func writeCounter(b *bytes.Buffer, name, help string, v int64) {
fmt.Fprintf(b, "# HELP %s %s\n# TYPE %s counter\n%s %d\n", name, help, name, name, v)
}
func writeGauge(b *bytes.Buffer, name, help string, v any) {
fmt.Fprintf(b, "# HELP %s %s\n# TYPE %s gauge\n%s %v\n", name, help, name, name, v)
}
// render is the text of /metrics (Prometheus exposition format).
func (s *Server) renderMetrics() string {
m := s.met
var b bytes.Buffer
now := s.now()
fmt.Fprintf(&b, "# HELP all_tunnel_build_info The node's build.\n# TYPE all_tunnel_build_info gauge\nall_tunnel_build_info{version=%q} 1\n", version)
writeGauge(&b, "all_tunnel_uptime_seconds", "Seconds since the node started.", int64(now.Sub(s.startedAt).Seconds()))
writeGauge(&b, "all_tunnel_goroutines", "Goroutines.", runtime.NumGoroutine())
writeGauge(&b, "all_tunnel_streams_open", "Client streams being served now.", s.streams.Load())
writeCounter(&b, "all_tunnel_streams_opened_total", "Client streams served since start.", m.streamsOpened.Load())
writeGauge(&b, "all_tunnel_control_connections", "Control connections of devices, authenticated or not yet.", s.controlCount())
writeGauge(&b, "all_tunnel_devices_online", "Devices online (one number: no list of them exists in the metrics).", s.routes.onlineCount())
writeGauge(&b, "all_tunnel_unauthenticated_connections", "Connections that have not finished their first message.", s.unauth.Load())
writeGauge(&b, "all_tunnel_pump_big_buffers", "32 KiB buffers held by a stream's pump now (an idle stream holds none).", s.bigBuffersInUse.Load())
writeGauge(&b, "all_tunnel_unauthenticated_anonymous_connections", "Of those, the ones whose source is not ChatGPT's, not a device's and not marked good (they share what the reserve leaves).", s.anonUnauth.Load())
writeGauge(&b, "all_tunnel_pending_attach", "OPENs waiting for their ATTACH.", s.pendingCount())
writeGauge(&b, "all_tunnel_draining", "1 while the node drains (SIGTERM).", boolNum(s.draining.Load()))
writeGauge(&b, "all_tunnel_kill_switch", "1 while the control plane's kill switch is on.", boolNum(s.killed.Load()))
m.rejected.write(&b)
m.streamEnd.write(&b)
m.bytes.write(&b)
m.authorize.write(&b)
m.feedEvents.write(&b)
size, banned, bans, overflowed := s.ips.snapshot()
writeGauge(&b, "all_tunnel_ip_table_entries", "Sources remembered (a number: no address leaves the node).", size)
writeGauge(&b, "all_tunnel_ip_banned", "Sources dropped at accept now for trying too many names.", banned)
writeCounter(&b, "all_tunnel_ip_bans_total", "Sources banned since start.", int64(bans))
writeCounter(&b, "all_tunnel_ip_table_overflow_total", "Sources that did not fit in the table and shared the overflow entry.", int64(overflowed))
writeGauge(&b, "all_tunnel_hint_addresses", "Addresses devices marked good, all devices together.", s.hints.size())
writeCounter(&b, "all_tunnel_hints_total", "HINT messages accepted.", m.hintsMarked.Load())
writeGauge(&b, "all_tunnel_gated_records", "Device hosts with a gated-day record.", s.gates.size())
writeCounter(&b, "all_tunnel_gated_cut_total", "Streams cut because a gated device's byte budget ran out.", m.gatedCut.Load())
writeGauge(&b, "all_tunnel_openai_prefixes", "Prefixes in the OpenAI list in use.", s.openai.size())
writeGauge(&b, "all_tunnel_openai_prefix_age_seconds", "Seconds since the OpenAI list in use was installed (-1: none).", int64(s.openai.age().Seconds()))
m.prefixRefresh.write(&b)
if s.cfg.CertNotAfter != nil {
if na := s.cfg.CertNotAfter(); !na.IsZero() {
writeGauge(&b, "all_tunnel_node_cert_not_after_seconds", "The end of the node's own certificate, Unix seconds (the monitor alerts under 14 and 5 days).", na.Unix())
writeGauge(&b, "all_tunnel_node_cert_seconds_left", "Seconds the node's own certificate has left.", int64(na.Sub(now).Seconds()))
}
}
if kh, ok := s.cfg.Keys.(interface{ Age() time.Duration }); ok {
writeGauge(&b, "all_tunnel_jwks_age_seconds", "Seconds since the control plane's keys were last fetched (-1: never).", int64(kh.Age().Seconds()))
}
if kh, ok := s.cfg.Keys.(keyHealth); ok {
writeGauge(&b, "all_tunnel_jwks_healthy", "1 while the node can vouch for the control plane's keys.", boolNum(kh.Healthy()))
}
writeGauge(&b, "all_tunnel_control_plane_down", "1 while nothing has answered from the control plane for the policy's quiet period.", boolNum(s.health.down(now, s.cfg.DownQuiet)))
writeCounter(&b, "all_tunnel_tls_client_hellos_observed_total", "ClientHellos the handshake observer read.", m.obsHellos.Load())
m.obsOffered.write(&b)
writeCounter(&b, "all_tunnel_tls_client_ech_total", "ClientHellos that carried an ECH extension.", m.obsECH.Load())
m.obsVersion.write(&b)
m.obsCipher.write(&b)
m.obsGroup.write(&b)
writeCounter(&b, "all_tunnel_tls_hello_retry_total", "HelloRetryRequests seen from devices.", m.obsRetry.Load())
writeCounter(&b, "all_tunnel_tls_server_hello_unparsed_total", "Streams whose first bytes from the device were not a ServerHello the observer follows.", m.obsUnparsed.Load())
return b.String()
}
func boolNum(v bool) int {
if v {
return 1
}
return 0
}
// healthState is the answer of /healthz: whether the node serves, in one word. A node that drains, has its kill switch on, or cannot vouch for the control
// plane's keys is not serving new devices, and says so with 503; a control plane that is down is reported but is 200, because what is connected goes on.
func (s *Server) healthState() (code int, body map[string]any) {
state := "ok"
switch {
case s.draining.Load():
state = "draining"
case s.killed.Load():
state = "kill_switch"
}
if kh, ok := s.cfg.Keys.(keyHealth); ok && state == "ok" && !kh.Healthy() {
state = "keys_stale"
}
code = http.StatusOK
if state != "ok" {
code = http.StatusServiceUnavailable
}
return code, map[string]any{"ok": state == "ok", "state": state, "control_plane_down": s.health.down(s.now(), s.cfg.DownQuiet)}
}
// metricsServer is the loopback HTTP listener of /metrics and /healthz.
type metricsServer struct {
ln net.Listener
srv *http.Server
}
func (s *Server) startMetrics() error {
if s.cfg.MetricsAddr == "" {
return nil
}
ln, err := net.Listen("tcp", s.cfg.MetricsAddr)
if err != nil {
return fmt.Errorf("metrics listener: %w", err)
}
mux := http.NewServeMux()
handle := func(path string, serve func(w http.ResponseWriter)) {
mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet && r.Method != http.MethodHead {
w.Header().Set("allow", "GET, HEAD")
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
if host, _, err := net.SplitHostPort(r.RemoteAddr); err != nil || !net.ParseIP(host).IsLoopback() {
http.NotFound(w, r) // the listener is loopback already; a connection that is not is not served
return
}
serve(w)
})
}
handle("/metrics", func(w http.ResponseWriter) {
w.Header().Set("content-type", "text/plain; version=0.0.4; charset=utf-8")
_, _ = w.Write([]byte(s.renderMetrics()))
})
handle("/healthz", func(w http.ResponseWriter) {
code, body := s.healthState()
raw, _ := json.Marshal(body)
w.Header().Set("content-type", "application/json")
w.WriteHeader(code)
_, _ = w.Write(append(raw, '\n'))
})
srv := &http.Server{Handler: mux, ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 10 * time.Second, WriteTimeout: 10 * time.Second, MaxHeaderBytes: 4 << 10}
s.metricsSrv = &metricsServer{ln: ln, srv: srv}
go func() { _ = srv.Serve(ln) }()
return nil
}
func (s *Server) stopMetrics() {
if s.metricsSrv == nil {
return
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
_ = s.metricsSrv.srv.Shutdown(ctx)
_ = s.metricsSrv.srv.Close()
}
// MetricsAddr is where /metrics listens ("" when it does not).
func (s *Server) MetricsAddr() string {
if s.metricsSrv == nil {
return ""
}
return s.metricsSrv.ln.Addr().String()
}
// controlCount is the number of control connections.
func (s *Server) controlCount() int {
s.mu.Lock()
defer s.mu.Unlock()
return len(s.conns)
}
files/tunnel/metrics_test.go 248 lines · 8.8 KB
package main
import (
"net/http"
"regexp"
"strconv"
"strings"
"testing"
"time"
)
// /metrics and /healthz (metrics.go): loopback, aggregate, no series per device.
func metricsEnv(t *testing.T, mutate func(*Config)) *testEnv {
return newTestEnv(t, func(c *Config) {
c.MetricsAddr = "127.0.0.1:0"
if mutate != nil {
mutate(c)
}
})
}
func TestMetrics_ServedOnLoopbackToGETAndHEADOnly(t *testing.T) {
e := metricsEnv(t, nil)
addr := e.srv.MetricsAddr()
if !strings.HasPrefix(addr, "127.0.0.1:") {
t.Fatalf("listening on %q", addr)
}
code, body, header := httpGet(t, "http://"+addr+"/metrics")
if code != 200 || !strings.HasPrefix(header.Get("content-type"), "text/plain") || !strings.Contains(body, "# TYPE all_tunnel_streams_open gauge") {
t.Fatalf("%d %v\n%s", code, header, body)
}
if code, _, _ := httpGet(t, "http://"+addr+"/nothing"); code != 404 {
t.Fatalf("an unknown path: %d", code)
}
for _, method := range []string{http.MethodPost, http.MethodPut, http.MethodDelete} {
req, _ := http.NewRequest(method, "http://"+addr+"/metrics", strings.NewReader("x"))
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
resp.Body.Close()
if resp.StatusCode != http.StatusMethodNotAllowed {
t.Fatalf("%s: %d", method, resp.StatusCode)
}
}
req, _ := http.NewRequest(http.MethodHead, "http://"+addr+"/healthz", nil)
resp, err := http.DefaultClient.Do(req)
if err != nil || resp.StatusCode != 200 {
t.Fatalf("HEAD /healthz: %v %v", resp, err)
}
resp.Body.Close()
// without the option there is no listener at all
if off := newTestEnv(t, nil); off.srv.MetricsAddr() != "" {
t.Fatalf("a node without -metrics-addr listens on %q", off.srv.MetricsAddr())
}
}
func TestMetrics_HealthzSaysWhetherTheNodeServes(t *testing.T) {
e := metricsEnv(t, nil)
url := "http://" + e.srv.MetricsAddr() + "/healthz"
if code, body, _ := httpGet(t, url); code != 200 || !strings.Contains(body, `"state":"ok"`) {
t.Fatalf("%d %s", code, body)
}
e.srv.setKilled(true)
if code, body, _ := httpGet(t, url); code != 503 || !strings.Contains(body, `"state":"kill_switch"`) {
t.Fatalf("%d %s", code, body)
}
e.srv.setKilled(false)
d := e.newDevice()
dc := e.connectOK(d)
release := make(chan struct{})
defer close(release)
dc.serve(holdHandler(d, release))
if _, err := e.clientTLS(d); err != nil { // a stream that stays open: the drain has something to wait for
t.Fatal(err)
}
go e.srv.Drain(10 * time.Second)
eventually(t, "draining", 3*time.Second, func() bool { return e.srv.Draining() })
if code, body, _ := httpGet(t, url); code != 503 || !strings.Contains(body, `"state":"draining"`) {
t.Fatalf("%d %s", code, body)
}
if v := e.metricValue("all_tunnel_draining"); v != 1 {
t.Fatalf("draining gauge %d", v)
}
}
// seriesOf is the set of series of a scrape without their values: name and label.
func seriesOf(text string) map[string]bool {
out := map[string]bool{}
for _, line := range strings.Split(text, "\n") {
if line == "" || strings.HasPrefix(line, "#") {
continue
}
if i := strings.LastIndexByte(line, ' '); i > 0 {
out[line[:i]] = true
}
}
return out
}
func TestMetrics_ThereIsNoSeriesPerLabelDeviceOrAddress(t *testing.T) {
scrape := func(devices int) string {
// (the devices of the test all connect from loopback, which is one source: its rate is out of the way)
e := newVIPEnv(t, func(e *testEnv, c *Config) { c.PerIPNewPerS, c.PerIPConcurrent = 1<<20, 1<<20 })
var held []*devConn
for i := 0; i < devices; i++ {
d := e.newDevice()
dc := e.connectOK(d)
dc.serve(nil)
held = append(held, dc)
c, err := e.clientFrom(randomIP(i), d)
if err != nil {
t.Fatalf("%v\n%s", err, e.logs.String())
}
if _, err := postOver(c, d.host, []byte("x")); err != nil {
t.Fatalf("%v\n%s", err, e.logs.String())
}
e.helloFrom(randomIP(i+500), "nobody"+d.label[6:]+".devices.all-e2e.test", time.Second) // a name that is not a device
dc.send("HINT " + randomIP(i+900))
}
text := e.srv.renderMetrics()
if devices > 0 {
if got := e.metricValue("all_tunnel_devices_online"); got != int64(devices) {
t.Fatalf("online %d", got)
}
}
_ = held
return text
}
one, many := scrape(1), scrape(25)
a, b := seriesOf(one), seriesOf(many)
if len(a) != len(b) {
t.Fatalf("%d series with one device, %d with 25", len(a), len(b))
}
for s := range b {
if !a[s] {
t.Errorf("series %s exists only with more devices", s)
}
}
for name, re := range map[string]*regexp.Regexp{
"a label": regexp.MustCompile(`[a-z2-7]{20}`),
"a zone": regexp.MustCompile(`devices\.all-e2e|\.test`),
"an address": regexp.MustCompile(`\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b|[0-9a-f]*:[0-9a-f]*:[0-9a-f:]+`),
"a device id": regexp.MustCompile(`[0-9a-f]{32}`),
"a stream id": regexp.MustCompile(`stream`),
} {
if name == "a stream id" {
continue // the word "stream" is in metric names; stream ids are the 32-hex check above
}
if m := re.FindString(many); m != "" {
t.Errorf("the metrics hold %s: %q", name, m)
}
}
// every label key of every series is one of a fixed few
keys := regexp.MustCompile(`\{([a-z_]+)=`)
allowed := map[string]bool{"reason": true, "direction": true, "result": true, "type": true, "group": true, "version": true, "cipher": true}
for s := range b {
if m := keys.FindStringSubmatch(s); m != nil && !allowed[m[1]] && !strings.HasPrefix(s, "all_tunnel_build_info") {
t.Errorf("series %s has a label key that is not in the closed list", s)
}
}
}
func randomIP(i int) string { return "198.51." + itoa(100+i/250) + "." + itoa(1+i%250) }
func itoa(n int) string { return strconv.Itoa(n) }
func TestMetrics_TheNodeCertificateAndTheKeysAreReported(t *testing.T) {
end := time.Now().Add(30 * 24 * time.Hour).Truncate(time.Second)
e, _ := httpNode(t, nil, func(c *Config) { c.CertNotAfter = func() time.Time { return end } })
if got := e.metricValue("all_tunnel_node_cert_not_after_seconds"); got != end.Unix() {
t.Fatalf("not_after %d, want %d", got, end.Unix())
}
left := e.metricValue("all_tunnel_node_cert_seconds_left")
if left < 29*24*3600 || left > 30*24*3600 {
t.Fatalf("seconds left %d", left)
}
eventually(t, "the keys were fetched", 5*time.Second, func() bool { return e.metricValue("all_tunnel_jwks_healthy") == 1 })
if age := e.metricValue("all_tunnel_jwks_age_seconds"); age < 0 || age > 60 {
t.Fatalf("age %d", age)
}
// without a certificate source of the kind that reports, the series is simply absent (a test node's fake keys report no age either)
plain := newTestEnv(t, nil)
if strings.Contains(plain.srv.renderMetrics(), "node_cert_not_after") || strings.Contains(plain.srv.renderMetrics(), "jwks_age") {
t.Fatal("series reported without a source")
}
}
func TestMetrics_CountersFollowWhatHappens(t *testing.T) {
e := metricsEnv(t, nil)
d := e.newDevice()
dc := e.connectOK(d)
dc.serve(nil)
c, err := e.clientTLS(d)
if err != nil {
t.Fatal(err)
}
if _, err := postOver(c, d.host, make([]byte, 5000)); err != nil {
t.Fatal(err)
}
c.Close()
e.waitIdle()
if e.metricValue("all_tunnel_streams_opened_total") != 1 || e.metricValue(`all_tunnel_authorize_total{result="ok"}`) != 1 {
t.Fatalf("\n%s", e.srv.renderMetrics())
}
if e.metricValue(`all_tunnel_bytes_total{direction="up"}`) < 5000 || e.metricValue(`all_tunnel_bytes_total{direction="down"}`) < 5000 {
t.Fatalf("bytes\n%s", e.srv.renderMetrics())
}
if e.metricValue("all_tunnel_control_connections") != 1 || e.metricValue("all_tunnel_devices_online") != 1 {
t.Fatalf("\n%s", e.srv.renderMetrics())
}
e.plan(d).err = &ControlError{Status: 402, Code: codeLicenseInactive}
other := e.newDeviceNoPlan()
oc := e.connect(other)
oc.hello(func() string { return e.session(other, nil) }, func(s, n string) string { return e.proof(other, s, n, nil) })
if e.metricValue(`all_tunnel_authorize_total{result="refused"}`) != 1 {
t.Fatalf("\n%s", e.srv.renderMetrics())
}
// something that is not TLS, and a name that is nobody's
e.helloFromLoopback("nobody.devices.all-e2e.test")
if e.rejected("unknown_host") != 1 {
t.Fatalf("\n%s", e.srv.renderMetrics())
}
}
func (e *testEnv) helloFromLoopback(host string) {
e.t.Helper()
c := sendRaw(e.t, e, helloOf(host, 0, 16000), 0)
buf := make([]byte, 16)
_ = c.SetReadDeadline(time.Now().Add(2 * time.Second))
_, _ = c.Read(buf)
}
func TestMetrics_TheTestNodeProcessServesThemAndLogsNameNobody(t *testing.T) {
p := launchTestNode(t, 3600, "-metrics-addr", "127.0.0.1:0")
addr, _ := p.ready["metrics"].(string)
if addr == "" {
t.Fatalf("the ready line has no metrics address: %v", p.ready)
}
code, body, _ := httpGet(t, "http://"+addr+"/metrics")
if code != 200 || !strings.Contains(body, "all_tunnel_node_cert_not_after_seconds") || !strings.Contains(body, `all_tunnel_build_info{version=`) {
t.Fatalf("%d\n%s", code, body)
}
if code, body, _ := httpGet(t, "http://"+addr+"/healthz"); code != 200 || !strings.Contains(body, `"ok":true`) {
t.Fatalf("%d %s", code, body)
}
}
files/tunnel/nftables_test.go 144 lines · 5.5 KB
package main
import (
"os"
"regexp"
"strconv"
"strings"
"testing"
)
// deploy/nftables.conf and deploy/sysctl.conf cannot be run on the machines the tests run on (no nftables, no root), so what is tested is what can be:
// that the kernel's limits are never stricter than the node's own (a retuned policy must not leave the kernel refusing what the node would serve), that
// the file only adds drops, and that it touches no port but the node's two.
func readDeploy(t *testing.T, name string) string {
t.Helper()
raw, err := os.ReadFile("deploy/" + name)
if err != nil {
t.Fatal(err)
}
return string(raw)
}
func nftLines(conf string) []string {
var out []string
for _, line := range strings.Split(conf, "\n") {
if line = strings.TrimSpace(line); line != "" && !strings.HasPrefix(line, "#") {
out = append(out, line)
}
}
return out
}
func TestNftables_TheKernelNeverRefusesWhatTheNodeWouldServe(t *testing.T) {
conf := readDeploy(t, "nftables.conf")
l := policyData.NodeLimits
factor := max(l.OpenAIPerIPFactor, l.DevicePerIPFactor)
perSourceHeld := l.PerIPConcurrent * factor
perSourceRate := l.PerIPNewPerS * factor
numbers := func(re string) []int {
var out []int
for _, m := range regexp.MustCompile(re).FindAllStringSubmatch(conf, -1) {
n, _ := strconv.Atoi(m[1])
out = append(out, n)
}
return out
}
held := numbers(`tcp dport 443 ct state new add @held[46] \{[^}]*ct count over (\d+)`)
rate := numbers(`tcp dport 443 ct state new add @rate[46] \{[^}]*limit rate over (\d+)/second`)
burst := numbers(`tcp dport 443 ct state new add @rate[46] \{[^}]*burst (\d+) packets`)
if len(held) != 2 || len(rate) != 2 || len(burst) != 2 {
t.Fatalf("the rules of :443 per source (IPv4 and IPv6 each): held %v rate %v burst %v", held, rate, burst)
}
for _, n := range held {
// the kernel counts a device's data connections too (one per open stream, the node frees its own slot at attach): room for one device at its cap
if n < l.LabelStreamsMax+1 {
t.Errorf("the kernel holds a source to %d connections, a device at its cap of %d streams holds %d", n, l.LabelStreamsMax, l.LabelStreamsMax+1)
}
if n < perSourceHeld {
t.Errorf("the kernel holds a source to %d connections, the node allows an OpenAI address %d", n, perSourceHeld)
}
}
for _, n := range rate {
if n < perSourceRate {
t.Errorf("the kernel allows %d new connections a second per source, the node allows an OpenAI address %d", n, perSourceRate)
}
}
for i, n := range burst {
if n < rate[i] {
t.Errorf("a burst of %d is smaller than the rate %d", n, rate[i])
}
}
// an IPv6 source is the policy's prefix: the mask of the rules is that many bits
mask := regexp.MustCompile(`ip6 saddr & ((?:[0-9a-f]{4}:)+:)`).FindStringSubmatch(conf)
if mask == nil || strings.Count(mask[1], "ffff") != l.IPv6PrefixBits/16 || l.IPv6PrefixBits%16 != 0 {
t.Errorf("the IPv6 mask %v is not the policy's %d bits", mask, l.IPv6PrefixBits)
}
// the whole node: the global flood limit is above everything the node could serve at once
if all := numbers(`tcp dport \{ 80, 443 \} ct state new limit rate over (\d+)/second`); len(all) != 1 || all[0] < l.GlobalStreamsMax {
t.Errorf("the global limit %v must exceed the node's %d streams", all, l.GlobalStreamsMax)
}
// port 80 serves one webroot: its limits are small, and the rules for it exist for IPv4 and IPv6
if h80 := numbers(`tcp dport 80 ct state new add @held80_[46] \{[^}]*ct count over (\d+)`); len(h80) != 2 || h80[0] > held[0] {
t.Errorf("port 80 holds %v", h80)
}
}
func TestNftables_LoadingTheFileTwiceDoesNotStackItsRules(t *testing.T) {
lines := nftLines(readDeploy(t, "nftables.conf"))
var ensure, drop, body = -1, -1, -1
for i, line := range lines {
switch {
case line == "table inet all_tunnel" && ensure < 0:
ensure = i
case line == "delete table inet all_tunnel":
drop = i
case line == "table inet all_tunnel {":
body = i
}
}
if ensure < 0 || drop != ensure+1 || body != drop+1 {
t.Fatalf("want 'table', 'delete table' and the table's body in a row, got lines %d, %d, %d", ensure, drop, body)
}
}
func TestNftables_OnlyAddsDropsAndTouchesOnlyThePortsOfTheNode(t *testing.T) {
conf := readDeploy(t, "nftables.conf")
if !strings.Contains(conf, "policy accept") || strings.Contains(conf, "policy drop") {
t.Fatal("the chain must accept by default: loading the fragment must never lock the operator out")
}
for _, line := range nftLines(conf) {
if !strings.Contains(line, "dport") {
continue
}
if !regexp.MustCompile(`^tcp dport (443|80|\{ 80, 443 \}) `).MatchString(line) {
t.Errorf("a rule on a port that is not the node's: %s", line)
}
if !strings.HasSuffix(line, "drop") {
t.Errorf("a rule that is not a drop: %s", line)
}
}
// every set the rules name is declared, and every set declared is used
declared := regexp.MustCompile(`set (\w+) \{`).FindAllStringSubmatch(conf, -1)
for _, m := range declared {
if strings.Count(conf, "@"+m[1]+" ") == 0 {
t.Errorf("set %s is declared and never used", m[1])
}
}
for _, m := range regexp.MustCompile(`@(\w+) `).FindAllStringSubmatch(conf, -1) {
if !strings.Contains(conf, "set "+m[1]+" {") {
t.Errorf("set %s is used and never declared", m[1])
}
}
if strings.Count(conf, "{")-strings.Count(conf, "}") != 0 {
t.Error("unbalanced braces")
}
}
func TestSysctl_SYNCookiesAreOn(t *testing.T) {
conf := readDeploy(t, "sysctl.conf")
if !regexp.MustCompile(`(?m)^net\.ipv4\.tcp_syncookies\s*=\s*1\s*$`).MatchString(conf) {
t.Fatal("net.ipv4.tcp_syncookies = 1 is not set")
}
}
files/tunnel/nodeproc_test.go 74 lines · 2.3 KB
package main
import (
"bufio"
"bytes"
"crypto/ed25519"
"crypto/rand"
"crypto/x509"
"encoding/json"
"os"
"os/exec"
"path/filepath"
"testing"
"time"
)
// nodeProc is the test-node binary running as a process, with a device the fake control plane knows.
type nodeProc struct {
e *testEnv
d *testDevice
cmd *exec.Cmd
exited chan error
logs *syncBuf
control string
feed string
ready map[string]any // the node's ready line
}
// launchTestNode starts the test node with one paid device (valid for validFor seconds) and the extra flags, and waits for its ready line.
func launchTestNode(t *testing.T, validFor int, extra ...string) *nodeProc {
t.Helper()
bin := testNodeBinary(t)
dir := t.TempDir()
pub, priv, _ := ed25519.GenerateKey(rand.Reader)
jwks, control, certOut := filepath.Join(dir, "jwks.json"), filepath.Join(dir, "control.json"), filepath.Join(dir, "node.pem")
jwksFile(t, jwks, testKID, pub)
e := &testEnv{t: t, cpKey: priv, tapByID: map[string]*bytes.Buffer{}}
e.srv = &Server{nodeURL: nodeProofURL(testNodeHost, 443)}
d := e.newDeviceNoPlan()
writeControl(t, control, map[string]any{d.id: map[string]any{"host": d.host, "status": "active", "plan": "paid", "valid_for_s": validFor}})
args := append([]string{"-https-addr", "127.0.0.1:0", "-http-addr", "127.0.0.1:0", "-node-id", testNodeID, "-node-host", testNodeHost,
"-test-jwks", jwks, "-test-control", control, "-test-cert-out", certOut, "-log-level", "info"}, extra...)
cmd := exec.Command(bin, args...)
stdout, _ := cmd.StdoutPipe()
p := &nodeProc{e: e, d: d, cmd: cmd, exited: make(chan error, 1), logs: &syncBuf{}, control: control}
cmd.Stderr = p.logs
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
go func() { p.exited <- cmd.Wait() }()
t.Cleanup(func() { _ = cmd.Process.Kill() })
ready := make(chan map[string]any, 1)
go func() {
line, _ := bufio.NewReader(stdout).ReadString('\n')
var m map[string]any
if json.Unmarshal([]byte(line), &m) == nil {
ready <- m
}
}()
select {
case info := <-ready:
e.httpsAddr = info["https"].(string)
p.ready = info
case <-time.After(20 * time.Second):
t.Fatalf("the test node did not announce itself:\n%s", p.logs.String())
}
certPEM, err := os.ReadFile(certOut)
if err != nil {
t.Fatal(err)
}
e.nodePool = x509.NewCertPool()
e.nodePool.AppendCertsFromPEM(certPEM)
return p
}
files/tunnel/observer.go 317 lines · 9.6 KB
package main
// The cleartext handshake observer (the design plan, sections 3 and 7). The node splices bytes it cannot read, but the first messages of a TLS
// handshake are cleartext by design, and what they say is the measurement G1 needs: which groups real ChatGPT clients offer (does the hybrid
// X25519MLKEM768 come?), and what the device's TLS server selects (version, cipher, group) so the plan's claims about strength are measured and not
// assumed. It reads three things and nothing else:
//
// - the ClientHello's supported_groups (and whether it carries an ECH extension), from the bytes the node already read to learn the SNI;
// - the ServerHello's version (supported_versions, else the legacy version), cipher suite and, in TLS 1.3, the key_share's selected group, from the
// first records the device sends back; a HelloRetryRequest is counted and the observer waits for the real ServerHello;
// - nothing after that: once the ServerHello is read (or the stream proves not to be TLS) the observer stops, and the rest of the stream is never
// looked at.
//
// What it keeps is a count per word of a CLOSED list (the tables below): never a name, an address, a label, a stream or a time. The words are the
// metrics' label values, so a client cannot make a new series by offering a new group: it is counted as "other". Nothing here is written to the log.
import "bytes"
const (
extSupportedGroups = 10
extSupportedVersions = 43
extKeyShare = 51
extECH = 0xfe0d // the RFC's code point (the draft ones are not counted)
)
// Words of the observer, in the order the metrics list them.
var (
obsGroupWords = []string{
"x25519", "secp256r1", "secp384r1", "secp521r1", "x448", "x25519mlkem768", "secp256r1mlkem768", "secp384r1mlkem1024", "x25519kyber768draft00", "ffdhe", "other",
}
obsVersionWords = []string{"tls13", "tls12", "tls11", "tls10", "other"}
obsCipherWords = []string{
"tls_aes_128_gcm_sha256", "tls_aes_256_gcm_sha384", "tls_chacha20_poly1305_sha256",
"ecdhe_ecdsa_aes_128_gcm_sha256", "ecdhe_ecdsa_aes_256_gcm_sha384", "ecdhe_rsa_aes_128_gcm_sha256", "ecdhe_rsa_aes_256_gcm_sha384",
"ecdhe_ecdsa_chacha20_poly1305", "ecdhe_rsa_chacha20_poly1305", "other",
}
// the selected group of a handshake that has none (TLS 1.2 here, or a ServerHello without a key share) is counted under this word
obsNoGroup = "none"
)
func isGREASE(v uint16) bool { return v&0x0f0f == 0x0a0a && byte(v>>8) == byte(v) }
// groupWord names a TLS named group.
func groupWord(g uint16) string {
switch g {
case 0x001d:
return "x25519"
case 0x0017:
return "secp256r1"
case 0x0018:
return "secp384r1"
case 0x0019:
return "secp521r1"
case 0x001e:
return "x448"
case 0x11ec:
return "x25519mlkem768"
case 0x11eb:
return "secp256r1mlkem768"
case 0x11ed:
return "secp384r1mlkem1024"
case 0x6399:
return "x25519kyber768draft00"
}
if g >= 0x0100 && g <= 0x01ff {
return "ffdhe"
}
return "other"
}
func versionWord(v uint16) string {
switch v {
case 0x0304:
return "tls13"
case 0x0303:
return "tls12"
case 0x0302:
return "tls11"
case 0x0301:
return "tls10"
}
return "other"
}
func cipherWord(c uint16) string {
switch c {
case 0x1301:
return "tls_aes_128_gcm_sha256"
case 0x1302:
return "tls_aes_256_gcm_sha384"
case 0x1303:
return "tls_chacha20_poly1305_sha256"
case 0xc02b:
return "ecdhe_ecdsa_aes_128_gcm_sha256"
case 0xc02c:
return "ecdhe_ecdsa_aes_256_gcm_sha384"
case 0xc02f:
return "ecdhe_rsa_aes_128_gcm_sha256"
case 0xc030:
return "ecdhe_rsa_aes_256_gcm_sha384"
case 0xcca9:
return "ecdhe_ecdsa_chacha20_poly1305"
case 0xcca8:
return "ecdhe_rsa_chacha20_poly1305"
}
return "other"
}
// helloShape is what the observer takes from a ClientHello.
type helloShape struct {
groups []uint16
ech bool
ok bool
}
// shapeOfClientHello reads a ClientHello body (the bytes after the 4-byte handshake header). It never reads outside body, and anything it cannot make
// sense of is simply not observed.
func shapeOfClientHello(body []byte) helloShape {
c := &cursor{b: body}
c.take(2) // legacy_version
c.take(32) // random
c.take(c.u8())
c.take(c.u16()) // cipher suites
c.take(c.u8()) // compression methods
if c.bad || len(c.b) < 2 {
return helloShape{}
}
ext := &cursor{b: c.take(c.u16())}
if c.bad {
return helloShape{}
}
shape := helloShape{ok: true}
for len(ext.b) > 0 {
typ := ext.u16()
data := &cursor{b: ext.take(ext.u16())}
if ext.bad {
return helloShape{}
}
switch typ {
case extSupportedGroups:
list := &cursor{b: data.take(data.u16())}
for len(list.b) >= 2 {
if g := uint16(list.u16()); !isGREASE(g) {
shape.groups = append(shape.groups, g)
}
}
case extECH:
shape.ech = true
}
}
return shape
}
// serverHello is what the observer takes from a ServerHello (or a HelloRetryRequest).
type serverHello struct {
version, cipher uint16
group uint16
hasGroup bool
retry bool
}
// helloRetryRandom is the fixed "random" that makes a ServerHello a HelloRetryRequest (RFC 8446 section 4.1.3).
var helloRetryRandom = []byte{
0xcf, 0x21, 0xad, 0x74, 0xe5, 0x9a, 0x61, 0x11, 0xbe, 0x1d, 0x8c, 0x02, 0x1e, 0x65, 0xb8, 0x91,
0xc2, 0xa2, 0x11, 0x16, 0x7a, 0xbb, 0x8c, 0x5e, 0x07, 0x9e, 0x09, 0xe2, 0xc8, 0xa8, 0x33, 0x9c,
}
// parseServerHello reads a ServerHello message body (after the 4-byte handshake header).
func parseServerHello(body []byte) (serverHello, bool) {
c := &cursor{b: body}
legacy := uint16(c.u16())
random := c.take(32)
c.take(c.u8()) // session id echo
cipher := uint16(c.u16())
c.take(1) // compression method
if c.bad {
return serverHello{}, false
}
sh := serverHello{version: legacy, cipher: cipher, retry: bytes.Equal(random, helloRetryRandom)}
if len(c.b) == 0 {
return sh, true // TLS 1.2 without extensions
}
ext := &cursor{b: c.take(c.u16())}
if c.bad {
return serverHello{}, false
}
for len(ext.b) > 0 {
typ := ext.u16()
data := &cursor{b: ext.take(ext.u16())}
if ext.bad {
return serverHello{}, false
}
switch typ {
case extSupportedVersions:
if v := uint16(data.u16()); !data.bad {
sh.version = v
}
case extKeyShare:
// ServerHello: the group, then the key exchange. HelloRetryRequest: the group alone.
if g := uint16(data.u16()); !data.bad {
sh.group, sh.hasGroup = g, true
}
}
}
return sh, true
}
// maxObserved is the most the observer reads of a stream's first bytes from the device before it gives up: a ServerHello is a hundred bytes, and a
// stream that has not shown one in this much is not a TLS handshake the observer understands.
const maxObserved = 16 << 10
// serverHelloWatch reads the device's first bytes of ONE client stream until the ServerHello has been seen, reporting to counters. It is used by one
// goroutine (the pump of that direction).
type serverHelloWatch struct {
m *metrics
buf []byte
seen int // hellos read: a retry request and the ServerHello after it make two
done bool
}
func newServerHelloWatch(m *metrics) *serverHelloWatch { return &serverHelloWatch{m: m} }
// feed gives the watcher the next bytes the device sent toward the client. After the ServerHello it costs one comparison.
func (w *serverHelloWatch) feed(p []byte) {
if w.done {
return
}
if len(w.buf)+len(p) > maxObserved {
w.giveUp()
return
}
w.buf = append(w.buf, p...)
for len(w.buf) > 0 {
// Only a handshake or a ChangeCipherSpec record (TLS 1.x) can come before the ServerHello: anything else is not one, said as soon as it shows.
if (w.buf[0] != 20 && w.buf[0] != 22) || (len(w.buf) >= 2 && w.buf[1] != 3) {
w.giveUp()
return
}
if len(w.buf) < 5 {
return
}
n := int(w.buf[3])<<8 | int(w.buf[4])
if len(w.buf) < 5+n {
return // the rest of the record is still on its way
}
typ, payload := w.buf[0], w.buf[5:5+n]
w.buf = w.buf[5+n:]
if typ == 22 && !w.handshake(payload) {
return
} // (a ChangeCipherSpec of the TLS 1.3 compatibility mode is not a hello: keep reading)
}
}
// handshake reads the hello a handshake record starts with. It reports whether the watcher goes on (a retry request is followed by a ServerHello).
func (w *serverHelloWatch) handshake(payload []byte) bool {
if len(payload) < 4 || payload[0] != 2 {
w.giveUp()
return false
}
l := int(payload[1])<<16 | int(payload[2])<<8 | int(payload[3])
if len(payload) < 4+l { // a ServerHello split over records: not seen in practice, not observed
w.giveUp()
return false
}
sh, ok := parseServerHello(payload[4 : 4+l])
if !ok {
w.giveUp()
return false
}
w.seen++
if sh.retry {
w.m.obsRetry.Add(1)
if w.seen >= 2 { // two retries are not a handshake
w.giveUp()
return false
}
return true
}
w.m.obsVersion.inc(versionWord(sh.version))
w.m.obsCipher.inc(cipherWord(sh.cipher))
if sh.hasGroup {
w.m.obsGroup.inc(groupWord(sh.group))
} else {
w.m.obsGroup.inc(obsNoGroup)
}
w.done, w.buf = true, nil
return false
}
// giveUp stops watching a stream whose first bytes were not a ServerHello the observer follows.
func (w *serverHelloWatch) giveUp() {
w.m.obsUnparsed.Add(1)
w.done, w.buf = true, nil
}
// observeClientHello counts what a ClientHello offered. peeked is the bytes the node read to find the SNI.
func (m *metrics) observeClientHello(peeked []byte) {
scan := scanClientHello(peeked)
if !scan.complete || scan.err != nil {
return
}
shape := shapeOfClientHello(scan.hello)
if !shape.ok {
return
}
m.obsHellos.Add(1)
seen := map[string]bool{} // a group counts once per ClientHello, however many times it is listed
for _, g := range shape.groups {
if w := groupWord(g); !seen[w] {
seen[w] = true
m.obsOffered.inc(w)
}
}
if shape.ech {
m.obsECH.Add(1)
}
}
files/tunnel/observer_test.go 379 lines · 13.7 KB
package main
import (
"bytes"
"context"
"crypto/tls"
"math/rand/v2"
"net"
"os/exec"
"strings"
"sync"
"testing"
"time"
)
// The cleartext handshake observer (observer.go): what the node reads of the first messages of a TLS handshake, and nothing else.
// recordedServerHello runs a TLS server with cfg against a client with ccfg over a pipe and returns the bytes the server sent first (until the client
// has what it needs): the device's side of a handshake, as the node would splice it.
func recordedServerHello(t testing.TB, scfg, ccfg *tls.Config) []byte {
t.Helper()
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer ln.Close()
var mu sync.Mutex
var sent bytes.Buffer
done := make(chan struct{})
go func() {
defer close(done)
raw, err := ln.Accept()
if err != nil {
return
}
defer raw.Close()
_ = raw.SetDeadline(time.Now().Add(5 * time.Second))
rec := &recordingConn{Conn: raw, onWrite: func(p []byte) { mu.Lock(); sent.Write(p); mu.Unlock() }}
_ = tls.Server(rec, scfg).Handshake()
}()
c, err := net.DialTimeout("tcp", ln.Addr().String(), 3*time.Second)
if err != nil {
t.Fatal(err)
}
_ = c.SetDeadline(time.Now().Add(5 * time.Second))
_ = tls.Client(c, ccfg).Handshake()
c.Close()
<-done
mu.Lock()
defer mu.Unlock()
return append([]byte(nil), sent.Bytes()...)
}
type recordingConn struct {
net.Conn
onWrite func([]byte)
}
func (r *recordingConn) Write(p []byte) (int, error) {
r.onWrite(p)
return r.Conn.Write(p)
}
func testServerConfig(t testing.TB) *tls.Config {
t.Helper()
_, cert := makeCA(t, "observer.example.test")
return &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS12}
}
func TestObserver_ClientHelloGroupsAndECH(t *testing.T) {
hello := captureClientHello(t, &tls.Config{ServerName: "dev1.example.test", InsecureSkipVerify: true, CurvePreferences: []tls.CurveID{tls.X25519MLKEM768, tls.X25519, tls.CurveP256}})
scan := scanClientHello(hello)
shape := shapeOfClientHello(scan.hello)
if !shape.ok || shape.ech {
t.Fatalf("%+v", shape)
}
words := map[string]bool{}
for _, g := range shape.groups {
words[groupWord(g)] = true
}
for _, want := range []string{"x25519mlkem768", "x25519", "secp256r1"} {
if !words[want] {
t.Errorf("offered groups %v lack %s", shape.groups, want)
}
}
m := newMetrics()
m.observeClientHello(hello)
if m.obsHellos.Load() != 1 || m.obsOffered.get("x25519mlkem768") != 1 || m.obsOffered.get("x25519") != 1 || m.obsECH.Load() != 0 {
t.Fatalf("counters: %d hellos, %d hybrid", m.obsHellos.Load(), m.obsOffered.get("x25519mlkem768"))
}
// a hello that is cut short, or not a hello, is not observed
m.observeClientHello(hello[:len(hello)/2])
m.observeClientHello([]byte("GET / HTTP/1.1\r\n"))
if m.obsHellos.Load() != 1 {
t.Fatal("an incomplete hello was observed")
}
}
func TestObserver_ServerHelloOfEveryKindOfHandshake(t *testing.T) {
cases := map[string]struct {
server, client *tls.Config
version, cipher, group string
retries int64
}{
"tls 1.3 with the hybrid group": {
&tls.Config{}, &tls.Config{CurvePreferences: []tls.CurveID{tls.X25519MLKEM768, tls.X25519}},
"tls13", "tls_aes_128_gcm_sha256", "x25519mlkem768", 0,
},
"tls 1.3 with x25519 only": {
&tls.Config{}, &tls.Config{CurvePreferences: []tls.CurveID{tls.X25519}},
"tls13", "tls_aes_128_gcm_sha256", "x25519", 0,
},
"tls 1.3 with chacha": {
&tls.Config{CipherSuites: nil}, &tls.Config{CurvePreferences: []tls.CurveID{tls.CurveP256}},
"tls13", "tls_aes_128_gcm_sha256", "secp256r1", 0,
},
"tls 1.2": {
&tls.Config{}, &tls.Config{MaxVersion: tls.VersionTLS12},
"tls12", "ecdhe_ecdsa_aes_128_gcm_sha256", "none", 0,
},
"a hello retry request": {
&tls.Config{CurvePreferences: []tls.CurveID{tls.CurveP256}}, &tls.Config{CurvePreferences: []tls.CurveID{tls.X25519, tls.CurveP256}},
"tls13", "tls_aes_128_gcm_sha256", "secp256r1", 1,
},
}
for name, c := range cases {
t.Run(name, func(t *testing.T) {
scfg := testServerConfig(t)
scfg.CurvePreferences, scfg.CipherSuites = c.server.CurvePreferences, c.server.CipherSuites
ccfg := &tls.Config{ServerName: "observer.example.test", InsecureSkipVerify: true, CurvePreferences: c.client.CurvePreferences, MaxVersion: c.client.MaxVersion}
stream := recordedServerHello(t, scfg, ccfg)
// the same bytes, whole and one at a time, give the same reading
for _, chunk := range []int{len(stream), 1, 7} {
m := newMetrics()
w := newServerHelloWatch(m)
for off := 0; off < len(stream); off += chunk {
w.feed(stream[off:min(off+chunk, len(stream))])
}
if m.obsVersion.get(c.version) != 1 || m.obsCipher.get(c.cipher) != 1 || m.obsGroup.get(c.group) != 1 || m.obsRetry.Load() != c.retries || m.obsUnparsed.Load() != 0 {
t.Fatalf("chunk %d: version %d cipher %d group %d retries %d unparsed %d\n%s", chunk, m.obsVersion.get(c.version), m.obsCipher.get(c.cipher),
m.obsGroup.get(c.group), m.obsRetry.Load(), m.obsUnparsed.Load(), m.renderGroups())
}
if !w.done {
t.Fatal("the watcher kept reading after the ServerHello")
}
}
})
}
}
func (m *metrics) renderGroups() string {
var b bytes.Buffer
m.obsVersion.write(&b)
m.obsCipher.write(&b)
m.obsGroup.write(&b)
return b.String()
}
func TestObserver_ReadsNothingAfterTheServerHelloAndGivesUpOnWhatIsNotOne(t *testing.T) {
scfg := testServerConfig(t)
stream := recordedServerHello(t, scfg, &tls.Config{ServerName: "observer.example.test", InsecureSkipVerify: true})
m := newMetrics()
w := newServerHelloWatch(m)
w.feed(stream)
if m.obsVersion.get("tls13") != 1 {
t.Fatal("not read")
}
before := len(w.buf)
w.feed(bytes.Repeat([]byte{22, 3, 3, 0, 4, 2, 0, 0, 0}, 100)) // more "hellos": never looked at
if m.obsVersion.get("tls13") != 1 || len(w.buf) != before {
t.Fatal("the watcher read on after the ServerHello")
}
for name, first := range map[string][]byte{
"an HTTP answer": []byte("HTTP/1.1 200 OK\r\n\r\n"),
"an alert": {21, 3, 3, 0, 2, 2, 40},
"application data": {23, 3, 3, 0, 5, 1, 2, 3, 4, 5},
"a ServerHello that lies": {22, 3, 3, 0, 8, 2, 0, 0, 40, 1, 2, 3, 4},
"a certificate where a hello": {22, 3, 3, 0, 6, 11, 0, 0, 2, 0, 0},
} {
m := newMetrics()
w := newServerHelloWatch(m)
w.feed(first)
if m.obsUnparsed.Load() != 1 || !w.done {
t.Errorf("%s: unparsed %d, done %v", name, m.obsUnparsed.Load(), w.done)
}
w.feed(first)
if m.obsUnparsed.Load() != 1 {
t.Errorf("%s: counted twice", name)
}
}
// a stream that never shows a hello in the observer's budget of bytes stops being watched
m2 := newMetrics()
w2 := newServerHelloWatch(m2)
for i := 0; i < 20 && !w2.done; i++ {
w2.feed(append([]byte{22, 3, 3, 0x03, 0xe8}, make([]byte, 500)...)) // a record that never completes
}
if !w2.done || m2.obsUnparsed.Load() != 1 || len(w2.buf) > maxObserved {
t.Fatalf("done %v unparsed %d buffered %d", w2.done, m2.obsUnparsed.Load(), len(w2.buf))
}
}
func TestObserver_NeverPanicsAndNeverReadsOutsideItsInput(t *testing.T) {
rng := rand.New(rand.NewPCG(7, 11))
hello := captureClientHello(t, &tls.Config{ServerName: "dev1.example.test", InsecureSkipVerify: true})
scfg := testServerConfig(t)
server := recordedServerHello(t, scfg, &tls.Config{ServerName: "observer.example.test", InsecureSkipVerify: true})
m := newMetrics()
for i := 0; i < 3000; i++ {
b := make([]byte, rng.IntN(400))
for j := range b {
b[j] = byte(rng.IntN(256))
}
shapeOfClientHello(b)
_, _ = parseServerHello(b)
newServerHelloWatch(m).feed(b)
// a real hello with bytes flipped and with its tail cut
mut := append([]byte(nil), hello...)
for k := 0; k < 3; k++ {
mut[rng.IntN(len(mut))] ^= byte(1 << rng.IntN(8))
}
m.observeClientHello(mut[:rng.IntN(len(mut))+1])
sv := append([]byte(nil), server...)
sv[rng.IntN(len(sv))] ^= byte(1 << rng.IntN(8))
w := newServerHelloWatch(m)
w.feed(sv[:rng.IntN(len(sv))+1])
}
}
func TestObserver_GREASEIsNotCountedAndWordsAreAClosedList(t *testing.T) {
for _, g := range []uint16{0x0a0a, 0x1a1a, 0xfafa} {
if !isGREASE(g) {
t.Errorf("%#x is GREASE", g)
}
}
if isGREASE(0x001d) || isGREASE(0x11ec) {
t.Error("a real group was taken for GREASE")
}
m := newMetrics()
m.obsOffered.inc("a group nobody has heard of")
m.obsOffered.inc("another")
if m.obsOffered.get("other") != 2 {
t.Fatal("an unknown word is counted as other")
}
if groupWord(0x1234) != "other" || cipherWord(0x1234) != "other" || versionWord(0x7f1c) != "other" || groupWord(0x0100) != "ffdhe" {
t.Fatal("the word tables")
}
}
func TestNode_ObserverReportsTLSVersionCipherAndGroupForAGoClientIncludingTheHybrid(t *testing.T) {
e := newTestEnv(t, nil)
d := e.newDevice()
dc := e.connectOK(d)
dc.serve(nil)
c, err := tls.DialWithDialer(&net.Dialer{Timeout: 3 * time.Second}, "tcp", e.httpsAddr, &tls.Config{
ServerName: d.host, RootCAs: d.pool, NextProtos: []string{"http/1.1"}, MinVersion: tls.VersionTLS13,
CurvePreferences: []tls.CurveID{tls.X25519MLKEM768, tls.X25519},
})
if err != nil {
t.Fatal(err)
}
if st := c.ConnectionState(); st.Version != tls.VersionTLS13 {
t.Fatalf("the Go handshake itself: version %#x", st.Version)
}
if _, err := postOver(c, d.host, []byte("x")); err != nil {
t.Fatal(err)
}
c.Close()
m := e.srv.met
if m.obsHellos.Load() != 1 || m.obsOffered.get("x25519mlkem768") != 1 || m.obsOffered.get("x25519") != 1 {
t.Fatalf("offered:\n%s", e.srv.renderMetrics())
}
if m.obsVersion.get("tls13") != 1 || m.obsCipher.get("tls_aes_128_gcm_sha256") != 1 || m.obsGroup.get("x25519mlkem768") != 1 {
t.Fatalf("selected:\n%s", e.srv.renderMetrics())
}
// the node's own conversation with a device (its name, its certificate) is not observed: it is the node's TLS, not a client's
if m.obsHellos.Load() != 1 {
t.Fatalf("the device's own connections were observed: %d", m.obsHellos.Load())
}
}
func openSSLWith(t *testing.T, group string) string {
t.Helper()
for _, bin := range []string{"/opt/homebrew/bin/openssl", "openssl"} {
path, err := exec.LookPath(bin)
if err != nil {
continue
}
out, err := exec.Command(path, "list", "-tls-groups").CombinedOutput()
if err == nil && strings.Contains(strings.ToLower(string(out)), strings.ToLower(group)) {
return path
}
}
return ""
}
func TestNode_ObserverReportsTheGroupsAnOpenSSLClientOffersAndWhatTheServerSelects(t *testing.T) {
bin := openSSLWith(t, "X25519MLKEM768")
if bin == "" {
t.Skip("no OpenSSL with X25519MLKEM768 on this machine")
}
for name, c := range map[string]struct {
groups string
offered []string
chosen string
}{
"the hybrid first": {"X25519MLKEM768:X25519", []string{"x25519mlkem768", "x25519"}, "x25519mlkem768"},
"x25519 and p-256": {"X25519:P-256", []string{"x25519", "secp256r1"}, "x25519"},
"p-256 only": {"P-256", []string{"secp256r1"}, "secp256r1"},
} {
t.Run(name, func(t *testing.T) {
e := newTestEnv(t, nil)
d := e.newDevice()
dc := e.connectOK(d)
dc.serve(nil)
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
cmd := exec.CommandContext(ctx, bin, "s_client", "-connect", e.httpsAddr, "-servername", d.host, "-tls1_3", "-groups", c.groups)
cmd.Stdin = strings.NewReader("")
out, _ := cmd.CombinedOutput()
if !strings.Contains(string(out), "TLSv1.3") {
t.Fatalf("OpenSSL did not complete a TLS 1.3 handshake:\n%s", out)
}
eventually(t, "the observer saw the handshake", 5*time.Second, func() bool { return e.srv.met.obsHellos.Load() == 1 && e.srv.met.obsVersion.get("tls13") == 1 })
for _, g := range c.offered {
if e.srv.met.obsOffered.get(g) != 1 {
t.Errorf("OpenSSL offered %s and the node did not count it:\n%s", g, e.srv.renderMetrics())
}
}
if e.srv.met.obsGroup.get(c.chosen) != 1 {
t.Errorf("the selected group should be %s:\n%s", c.chosen, e.srv.renderMetrics())
}
})
}
}
// helloWithExtensions builds a ClientHello for host with the extensions of the test (supported_groups, an ECH extension, GREASE).
func helloWithExtensions(host string, groups []uint16, ech bool) []byte {
ext := sniExt(host)
if len(groups) > 0 {
list := []byte{byte(len(groups) * 2 >> 8), byte(len(groups) * 2)}
for _, g := range groups {
list = append(list, byte(g>>8), byte(g))
}
ext = append(append([]byte{0, 10, byte(len(list) >> 8), byte(len(list))}, list...), ext...)
}
if ech {
ext = append([]byte{0xfe, 0x0d, 0, 4, 1, 2, 3, 4}, ext...)
}
var body []byte
body = append(body, 3, 3)
body = append(body, make([]byte, 32)...)
body = append(body, 0)
body = append(body, 0, 2, 0x13, 0x01)
body = append(body, 1, 0)
body = append(body, byte(len(ext)>>8), byte(len(ext)))
body = append(body, ext...)
hs := append([]byte{1, byte(len(body) >> 16), byte(len(body) >> 8), byte(len(body))}, body...)
return append([]byte{22, 3, 1, byte(len(hs) >> 8), byte(len(hs))}, hs...)
}
func TestNode_ObserverCountsAnECHExtensionGREASEAndAServerThatAnswersWithoutTLS(t *testing.T) {
e := newTestEnv(t, nil)
d := e.newDevice()
dc := e.connectOK(d)
dc.serve(func(c net.Conn) { _, _ = c.Write([]byte("HTTP/1.1 400 not tls\r\n\r\n")) })
raw := sendRaw(t, e, helloWithExtensions(d.host, []uint16{0x0a0a, 0x11ec, 0x001d, 0x001d, 0x9999}, true), 0)
_ = raw.SetReadDeadline(time.Now().Add(3 * time.Second))
buf := make([]byte, 64)
_, _ = raw.Read(buf)
eventually(t, "the hello observed", 3*time.Second, func() bool { return e.srv.met.obsHellos.Load() == 1 })
m := e.srv.met
if m.obsECH.Load() != 1 {
t.Fatalf("ECH: %d", m.obsECH.Load())
}
if m.obsOffered.get("x25519mlkem768") != 1 || m.obsOffered.get("x25519") != 1 || m.obsOffered.get("other") != 1 {
t.Fatalf("a group listed twice counts once, GREASE not at all, an unknown one as other:\n%s", e.srv.renderMetrics())
}
eventually(t, "the answer that was not a ServerHello", 3*time.Second, func() bool { return m.obsUnparsed.Load() == 1 })
}
files/tunnel/openai_prefixes.go 336 lines · 10.0 KB
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"net/netip"
"os"
"path/filepath"
"slices"
"sync/atomic"
"time"
)
// The address ranges OpenAI publishes for its connector traffic (the design plan, section 5). The node reads them once a day into the "openai"
// source class of an OPEN, which is a QUOTA and not an allowlist: ChatGPT can be made to call any name from those addresses, so the class never lets a
// stream past a limit; it only keeps ChatGPT's own share of the node (policy node_limits.openai_reserved_pct) and per-source limits larger by a factor
// (openai_per_ip_factor) for an address that serves many users. The published list is "best effort" (G1 measures the addresses ChatGPT really comes from).
//
// A list that is empty, corrupt, too big, or that holds a prefix too wide to be a connector's range (0.0.0.0/0 would make everyone "openai") replaces
// nothing: the node keeps the last good one, which is also kept in a file (public data) so that a restart during an outage of the publisher does not
// start with none.
// The words of a refresh's outcome, in the log and in the metrics.
const (
prefixOK = "ok"
prefixFetch = "fetch_failed"
prefixEmpty = "empty"
prefixCorrupt = "corrupt"
prefixTooLarge = "too_large"
prefixTooMany = "too_many"
prefixBadPrefix = "bad_prefix"
)
var (
errPrefixEmpty = errors.New(prefixEmpty)
errPrefixCorrupt = errors.New(prefixCorrupt)
errPrefixTooLarge = errors.New(prefixTooLarge)
errPrefixTooMany = errors.New(prefixTooMany)
errPrefixBad = errors.New(prefixBadPrefix)
)
// prefixSet is an immutable list of prefixes with a lookup that costs one map probe per distinct prefix length.
type prefixSet struct {
set map[netip.Prefix]struct{}
lens []int
}
func newPrefixSet(prefixes []netip.Prefix) *prefixSet {
p := &prefixSet{set: make(map[netip.Prefix]struct{}, len(prefixes))}
for _, pf := range prefixes {
p.set[pf] = struct{}{}
if !slices.Contains(p.lens, pf.Bits()) {
p.lens = append(p.lens, pf.Bits())
}
}
return p
}
func (p *prefixSet) len() int { return len(p.set) }
func (p *prefixSet) contains(a netip.Addr) bool {
a = a.WithZone("").Unmap()
for _, bits := range p.lens {
if pf, err := a.Prefix(bits); err == nil {
if _, ok := p.set[pf]; ok {
return true
}
}
}
return false
}
// prefixLimits are the sanity bounds a list is read under (policy node_limits).
type prefixLimits struct {
maxBytes, maxEntries, minV4, minV6 int
}
// parsePrefixes reads a published list: {"prefixes": [{"ipv4Prefix": "a.b.c.d/n"}, {"ipv6Prefix": "..."}]} and nothing else of consequence. Every entry
// must be one of those two shapes with a prefix of its own family, not narrower than the policy's minimum width (a connector range is a /28 or so, not a
// continent); a single bad entry refuses the whole list, because a list that is partly wrong is not one to trust the rest of.
func parsePrefixes(body []byte, lim prefixLimits) ([]netip.Prefix, error) {
if len(body) > lim.maxBytes {
return nil, errPrefixTooLarge
}
value, err := parseJSON(string(body))
if err != nil {
return nil, errPrefixCorrupt
}
doc, ok := value.(map[string]any)
if !ok {
return nil, errPrefixCorrupt
}
list, ok := doc["prefixes"].([]any)
if !ok {
return nil, errPrefixCorrupt
}
if len(list) == 0 {
return nil, errPrefixEmpty
}
if len(list) > lim.maxEntries {
return nil, errPrefixTooMany
}
out := make([]netip.Prefix, 0, len(list))
for _, item := range list {
entry, ok := item.(map[string]any)
if !ok || len(entry) != 1 {
return nil, errPrefixCorrupt
}
for key, raw := range entry {
text, ok := raw.(string)
if !ok || (key != "ipv4Prefix" && key != "ipv6Prefix") {
return nil, errPrefixCorrupt
}
pf, err := netip.ParsePrefix(text)
if err != nil || pf.Addr().Zone() != "" || pf.Addr().Is4In6() || pf.Addr().IsUnspecified() {
return nil, errPrefixBad
}
minBits := lim.minV6
if key == "ipv4Prefix" {
if !pf.Addr().Is4() {
return nil, errPrefixBad
}
minBits = lim.minV4
} else if !pf.Addr().Is6() {
return nil, errPrefixBad
}
if pf.Bits() < minBits {
return nil, errPrefixBad
}
out = append(out, pf.Masked())
}
}
return out, nil
}
// encodePrefixes writes a list in the format parsePrefixes reads (what the file of the last good list holds).
func encodePrefixes(set *prefixSet) []byte {
all := make([]netip.Prefix, 0, len(set.set))
for pf := range set.set {
all = append(all, pf)
}
slices.SortFunc(all, func(a, b netip.Prefix) int { return a.Addr().Compare(b.Addr()) })
entries := make([]map[string]string, 0, len(all))
for _, pf := range all {
key := "ipv6Prefix"
if pf.Addr().Is4() {
key = "ipv4Prefix"
}
entries = append(entries, map[string]string{key: pf.String()})
}
raw, _ := json.Marshal(map[string]any{"prefixes": entries})
return raw
}
// openaiPrefixes is the node's current list and what keeps it fresh.
type openaiPrefixes struct {
cur atomic.Pointer[prefixSet]
at atomic.Int64 // when the list in use was installed (clock nanoseconds since the epoch), 0 when none
clk clock
log *slog.Logger
lim prefixLimits
every time.Duration
file string
fetch func(ctx context.Context) ([]byte, error)
count func(result string) // the metrics' counter of outcomes
}
func newOpenAIPrefixes(cfg *Config, clk clock, count func(string)) *openaiPrefixes {
l := policyData.NodeLimits
o := &openaiPrefixes{
clk: clk, log: cfg.Logger, every: cfg.OpenAIPrefixRefresh, file: cfg.OpenAIPrefixesCache, count: count,
lim: prefixLimits{maxBytes: l.OpenAIPrefixBodyMaxBytes, maxEntries: l.OpenAIPrefixMax, minV4: l.OpenAIPrefixMinV4Bits, minV6: l.OpenAIPrefixMinV6Bits},
}
switch {
case cfg.PrefixFetch != nil:
o.fetch = cfg.PrefixFetch
case cfg.OpenAIPrefixesURL != "":
o.fetch = func(ctx context.Context) ([]byte, error) {
return httpFetchList(ctx, cfg.OpenAIPrefixesURL, int64(o.lim.maxBytes))
}
}
return o
}
// contains reports whether a is in the list in use (false when there is none).
func (o *openaiPrefixes) contains(a netip.Addr) bool {
if set := o.cur.Load(); set != nil {
return set.contains(a)
}
return false
}
// size is how many prefixes are in use, and age how long ago the list was installed (-1 when there is none).
func (o *openaiPrefixes) size() int {
if set := o.cur.Load(); set != nil {
return set.len()
}
return 0
}
func (o *openaiPrefixes) age() time.Duration {
at := o.at.Load()
if at == 0 {
return -1
}
return o.clk.Now().Sub(time.Unix(0, at))
}
// apply reads body and, when it is a good list, puts it in use and (unless it came from the file) keeps it in the file. A list that is not good
// changes nothing, and the error says why in one of the outcome words.
func (o *openaiPrefixes) apply(body []byte, fromFile bool) error {
prefixes, err := parsePrefixes(body, o.lim)
if err != nil {
return err
}
set := newPrefixSet(prefixes)
o.cur.Store(set)
o.at.Store(o.clk.Now().UnixNano())
if !fromFile && o.file != "" {
if werr := writeAtomic(o.file, encodePrefixes(set)); werr != nil {
o.log.Warn("openai_prefixes_not_kept", "reason", "write_failed")
}
}
return nil
}
// writeAtomic replaces path with data in one rename, so a crash never leaves half a list.
func writeAtomic(path string, data []byte) error {
tmp, err := os.CreateTemp(filepath.Dir(path), ".prefixes-*")
if err != nil {
return err
}
_, werr := tmp.Write(data)
cerr := tmp.Close()
if werr != nil || cerr != nil {
_ = os.Remove(tmp.Name())
return errors.Join(werr, cerr)
}
if err := os.Chmod(tmp.Name(), 0o644); err != nil {
_ = os.Remove(tmp.Name())
return err
}
if err := os.Rename(tmp.Name(), path); err != nil {
_ = os.Remove(tmp.Name())
return err
}
return nil
}
// refresh fetches and applies once, and says what came of it.
func (o *openaiPrefixes) refresh(ctx context.Context) string {
if o.fetch == nil {
return prefixFetch
}
fctx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
body, err := o.fetch(fctx)
result := prefixOK
switch {
case err != nil && errors.Is(err, errPrefixTooLarge):
result = prefixTooLarge
case err != nil:
result = prefixFetch
default:
if aerr := o.apply(body, false); aerr != nil {
result = aerr.Error()
}
}
o.count(result)
if result == prefixOK {
o.log.Info("openai_prefixes_refreshed", "prefixes", o.size())
} else {
o.log.Warn("openai_prefixes_refused", "reason", result, "keeping", o.size())
}
return result
}
// Run is the node's background task: the last good list from the file, then a refresh at start and every refresh period (a failed one is tried again
// after minutes, not a day), until the node ends.
func (o *openaiPrefixes) Run(ctx context.Context) {
if o.file != "" {
if body, err := os.ReadFile(o.file); err == nil {
if aerr := o.apply(body, true); aerr != nil {
o.log.Warn("openai_prefixes_file_refused", "reason", aerr.Error())
}
}
}
if o.fetch == nil {
return
}
failures := 0
for {
wait := o.every
if o.refresh(ctx) == prefixOK {
failures = 0
} else {
failures++
wait = min(5*time.Minute<<min(failures-1, 4), o.every)
}
if !o.clk.sleep(ctx, jitter(wait, 10)) {
return
}
}
}
// httpFetchList is the one request the node makes to the publisher: HTTPS, no redirect, no credentials, at most limit bytes.
func httpFetchList(ctx context.Context, url string, limit int64) ([]byte, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
req.Header.Set("accept", "application/json")
req.Header.Set("user-agent", "all-tunnel/"+version)
client := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("status %d", resp.StatusCode)
}
body, err := io.ReadAll(io.LimitReader(resp.Body, limit+1))
if err != nil {
return nil, err
}
if int64(len(body)) > limit {
return nil, errPrefixTooLarge
}
return body, nil
}
files/tunnel/openai_prefixes_test.go 323 lines · 12.0 KB
package main
import (
"context"
"errors"
"fmt"
"log/slog"
"net/http"
"net/http/httptest"
"net/netip"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
)
func testPrefixLimits() prefixLimits {
l := policyData.NodeLimits
return prefixLimits{maxBytes: l.OpenAIPrefixBodyMaxBytes, maxEntries: l.OpenAIPrefixMax, minV4: l.OpenAIPrefixMinV4Bits, minV6: l.OpenAIPrefixMinV6Bits}
}
func listOf(prefixes ...string) string {
var items []string
for _, p := range prefixes {
key := "ipv4Prefix"
if strings.Contains(p, ":") {
key = "ipv6Prefix"
}
items = append(items, fmt.Sprintf(`{%q:%q}`, key, p))
}
return `{"creationTime":"2026-10-01T00:00:00.000000","prefixes":[` + strings.Join(items, ",") + `]}`
}
func TestPrefixes_AGoodListIsReadAndLooksUpIPv4AndIPv6(t *testing.T) {
got, err := parsePrefixes([]byte(listOf("203.0.113.0/28", "198.51.100.64/26", "2001:db8:abcd::/48", "192.0.2.5/24")), testPrefixLimits())
if err != nil {
t.Fatal(err)
}
set := newPrefixSet(got)
for addr, want := range map[string]bool{
"203.0.113.7": true, "203.0.113.16": false, "198.51.100.100": true, "198.51.100.200": false, "2001:db8:abcd:1::9": true, "2001:db8:abce::1": false,
"192.0.2.99": true, // 192.0.2.5/24 is the /24
"::ffff:203.0.113.7": true, "198.51.100.8": false,
} {
if set.contains(netip.MustParseAddr(addr)) != want {
t.Errorf("%s: want %v", addr, want)
}
}
if set.len() != 4 {
t.Fatalf("%d prefixes", set.len())
}
}
func TestPrefixes_ACorruptEmptyHugeOrTooWideListIsRefused(t *testing.T) {
lim := testPrefixLimits()
manyItems := make([]string, lim.maxEntries+1)
for i := range manyItems {
manyItems[i] = fmt.Sprintf("10.%d.%d.0/24", i/256, i%256)
}
for name, c := range map[string]struct {
body string
want error
}{
"empty body": {"", errPrefixCorrupt},
"not json": {"<html>", errPrefixCorrupt},
"a list not an object": {`[{"ipv4Prefix":"203.0.113.0/28"}]`, errPrefixCorrupt},
"no prefixes key": {`{"x":[]}`, errPrefixCorrupt},
"an empty list": {`{"prefixes":[]}`, errPrefixEmpty},
"an entry that is a string": {`{"prefixes":["203.0.113.0/28"]}`, errPrefixCorrupt},
"an unknown key": {`{"prefixes":[{"cidr":"203.0.113.0/28"}]}`, errPrefixCorrupt},
"two keys in one entry": {`{"prefixes":[{"ipv4Prefix":"203.0.113.0/28","ipv6Prefix":"2001:db8::/48"}]}`, errPrefixCorrupt},
"a number as the prefix": {`{"prefixes":[{"ipv4Prefix":5}]}`, errPrefixCorrupt},
"not a prefix": {`{"prefixes":[{"ipv4Prefix":"203.0.113.0"}]}`, errPrefixBad},
"a v6 prefix under a v4 key": {`{"prefixes":[{"ipv4Prefix":"2001:db8::/48"}]}`, errPrefixBad},
"a v4 prefix under a v6 key": {`{"prefixes":[{"ipv6Prefix":"203.0.113.0/28"}]}`, errPrefixBad},
"everything": {listOf("0.0.0.0/0"), errPrefixBad},
"half the internet": {listOf("128.0.0.0/1"), errPrefixBad},
"a wide v4 range": {listOf("10.0.0.0/8"), errPrefixBad},
"a wide v6 range": {listOf("2000::/3"), errPrefixBad},
"a mapped address": {`{"prefixes":[{"ipv6Prefix":"::ffff:203.0.113.0/120"}]}`, errPrefixBad},
"two good prefixes": {listOf("203.0.113.0/28", "198.51.100.0/26") + "", nil},
"too many entries": {listOf(manyItems...), errPrefixTooMany},
"a body that is too big": {`{"prefixes":[],"pad":"` + strings.Repeat("x", lim.maxBytes) + `"}`, errPrefixTooLarge},
"a repeated key": {`{"prefixes":[{"ipv4Prefix":"203.0.113.0/28","ipv4Prefix":"198.51.100.0/26"}]}`, errPrefixCorrupt},
} {
_, err := parsePrefixes([]byte(c.body), lim)
if c.want == nil {
if err != nil {
t.Errorf("%s: %v", name, err)
}
continue
}
if !errors.Is(err, c.want) {
t.Errorf("%s: got %v, want %v", name, err, c.want)
}
}
// one bad entry refuses the whole list
if _, err := parsePrefixes([]byte(`{"prefixes":[{"ipv4Prefix":"203.0.113.0/28"},{"ipv4Prefix":"0.0.0.0/0"}]}`), lim); err == nil {
t.Error("a list with one too-wide prefix was taken")
}
}
type prefixHarness struct {
o *openaiPrefixes
fc *fakeClock
results sync.Map // outcome word -> *atomic.Int64
logs *syncBuf
file string
}
func (h *prefixHarness) count(word string) int64 {
if v, ok := h.results.Load(word); ok {
return v.(*atomic.Int64).Load()
}
return 0
}
func newPrefixHarness(t *testing.T, fetch func(context.Context) ([]byte, error)) *prefixHarness {
t.Helper()
h := &prefixHarness{fc: newFakeClock(), logs: &syncBuf{}, file: filepath.Join(t.TempDir(), "prefixes.json")}
cfg := DefaultConfig()
cfg.OpenAIPrefixesCache = h.file
cfg.PrefixFetch = fetch
cfg.Logger = slog.New(slog.NewJSONHandler(h.logs, nil))
h.o = newOpenAIPrefixes(&cfg, newClock(h.fc.Now, h.fc.AfterFunc), func(word string) {
v, _ := h.results.LoadOrStore(word, &atomic.Int64{})
v.(*atomic.Int64).Add(1)
})
return h
}
func TestPrefixes_ARefreshThatFailsKeepsTheLastGoodListAndAFileSurvivesARestart(t *testing.T) {
var mu sync.Mutex
body, fail := listOf("203.0.113.0/28"), error(nil)
h := newPrefixHarness(t, func(context.Context) ([]byte, error) {
mu.Lock()
defer mu.Unlock()
return []byte(body), fail
})
in := netip.MustParseAddr("203.0.113.5")
if h.o.contains(in) {
t.Fatal("a list that nobody read is in use")
}
if got := h.o.refresh(context.Background()); got != prefixOK || !h.o.contains(in) || h.o.size() != 1 {
t.Fatalf("first refresh: %s, size %d", got, h.o.size())
}
good := h.o.cur.Load()
for name, c := range map[string]struct {
body string
err error
want string
}{
"the publisher is down": {"", errors.New("connection refused"), prefixFetch},
"an empty list": {`{"prefixes":[]}`, nil, prefixEmpty},
"a corrupt body": {"{{{", nil, prefixCorrupt},
"a list of the internet": {listOf("0.0.0.0/0"), nil, prefixBadPrefix},
"a body past the limit": {"", errPrefixTooLarge, prefixTooLarge},
"a list with too many": {listOf(strings.Split(strings.Repeat("10.1.1.0/24,", 2049), ",")[:2049]...), nil, prefixTooMany},
"nothing but whitespace": {" ", nil, prefixCorrupt},
"a list of another shape": {`{"prefixes":[{"cidr":"203.0.113.0/28"}]}`, nil, prefixCorrupt},
} {
mu.Lock()
body, fail = c.body, c.err
mu.Unlock()
if got := h.o.refresh(context.Background()); got != c.want {
t.Errorf("%s: outcome %q, want %q", name, got, c.want)
}
if h.o.cur.Load() != good || !h.o.contains(in) {
t.Fatalf("%s: the list in use changed", name)
}
}
// a new good list replaces it, and the file holds it
mu.Lock()
body, fail = listOf("198.51.100.0/26", "2001:db8::/32"), nil
mu.Unlock()
if h.o.refresh(context.Background()) != prefixOK || h.o.contains(in) || !h.o.contains(netip.MustParseAddr("198.51.100.9")) {
t.Fatal("the new list was not taken")
}
// a restart with the publisher down: the file is the list
restarted := newPrefixHarness(t, func(context.Context) ([]byte, error) { return nil, errors.New("down") })
raw, err := os.ReadFile(h.file)
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(restarted.file, raw, 0o600); err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() { restarted.o.Run(ctx); close(done) }()
eventually(t, "the list from the file", 3*time.Second, func() bool { return restarted.o.size() == 2 })
if !restarted.o.contains(netip.MustParseAddr("2001:db8:1::1")) {
t.Fatal("the IPv6 prefix of the file is not in use")
}
eventually(t, "the failed refresh was counted", 3*time.Second, func() bool { return restarted.count(prefixFetch) >= 1 })
if restarted.o.size() != 2 {
t.Fatal("a failed refresh dropped the list of the file")
}
cancel()
<-done
if info, err := os.Stat(h.file); err != nil || info.Mode().Perm() != 0o644 {
t.Fatalf("the file: %v %v", info, err)
}
// a file that is corrupt is ignored
if err := os.WriteFile(restarted.file, []byte("garbage"), 0o600); err != nil {
t.Fatal(err)
}
again := newPrefixHarness(t, nil)
again.file = restarted.file
again.o.file = restarted.file
again.o.Run(context.Background())
if again.o.size() != 0 || !strings.Contains(again.logs.String(), "openai_prefixes_file_refused") {
t.Fatalf("size %d logs %s", again.o.size(), again.logs.String())
}
}
func TestPrefixes_RunRefreshesDailyAndRetriesAFailureSoonerNotAtOnce(t *testing.T) {
var calls atomic.Int64
var fail atomic.Bool
fail.Store(true)
h := newPrefixHarness(t, func(context.Context) ([]byte, error) {
calls.Add(1)
if fail.Load() {
return nil, errors.New("down")
}
return []byte(listOf("203.0.113.0/28")), nil
})
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go h.o.Run(ctx)
eventually(t, "the first attempt", 3*time.Second, func() bool { return calls.Load() == 1 })
// 4 minutes later nothing was retried (the retry waits minutes, not seconds: the publisher is not hammered)
h.fc.Advance(4 * time.Minute)
time.Sleep(50 * time.Millisecond)
if calls.Load() != 1 {
t.Fatalf("%d calls after 4 minutes", calls.Load())
}
fail.Store(false)
h.fc.Advance(2 * time.Minute)
eventually(t, "the retry and the good list", 3*time.Second, func() bool { return h.o.size() == 1 })
n := calls.Load()
// then the next refresh is a day later (give or take the jitter that keeps nodes from calling in step)
h.fc.Advance(20 * time.Hour)
time.Sleep(50 * time.Millisecond)
if calls.Load() != n {
t.Fatalf("refreshed again before the day was over (%d calls, was %d)", calls.Load(), n)
}
h.fc.Advance(8 * time.Hour)
eventually(t, "the daily refresh", 3*time.Second, func() bool { return calls.Load() == n+1 })
if age := h.o.age(); age < 0 || age > time.Minute {
t.Fatalf("age %s", age)
}
}
func TestPrefixes_TheHTTPFetchReadsOneUrlWithoutRedirectsAndWithinTheLimit(t *testing.T) {
var hits atomic.Int64
mux := http.NewServeMux()
mux.HandleFunc("/list.json", func(w http.ResponseWriter, r *http.Request) {
hits.Add(1)
if r.Header.Get("cookie") != "" || r.Header.Get("authorization") != "" {
t.Error("the fetch sent credentials")
}
_, _ = w.Write([]byte(listOf("203.0.113.0/28")))
})
mux.HandleFunc("/moved", func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "/list.json", http.StatusFound) })
mux.HandleFunc("/big", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write(make([]byte, 3000)) })
mux.HandleFunc("/missing", func(w http.ResponseWriter, r *http.Request) { http.NotFound(w, r) })
srv := httptest.NewServer(mux)
defer srv.Close()
ctx := context.Background()
if body, err := httpFetchList(ctx, srv.URL+"/list.json", 4096); err != nil || !strings.Contains(string(body), "203.0.113.0/28") {
t.Fatalf("%q %v", body, err)
}
if _, err := httpFetchList(ctx, srv.URL+"/moved", 4096); err == nil {
t.Fatal("a redirect was followed")
}
if _, err := httpFetchList(ctx, srv.URL+"/big", 1000); !errors.Is(err, errPrefixTooLarge) {
t.Fatalf("%v", err)
}
if _, err := httpFetchList(ctx, srv.URL+"/missing", 4096); err == nil {
t.Fatal("a 404 is not a list")
}
}
func TestNode_ASourceInTheOpenAIListIsClassedOpenAIAndGetsLargerPerSourceLimits(t *testing.T) {
release := make(chan struct{})
e := newVIPEnv(t, func(e *testEnv, c *Config) {
c.PerIPConcurrent, c.PerIPNewPerS, c.OpenAIPerIPFactor = 2, 1000, 4
c.PrefixFetch = func(context.Context) ([]byte, error) { return []byte(listOf("192.0.2.0/24")), nil }
})
eventually(t, "the list", 3*time.Second, func() bool { return e.srv.openai.size() == 1 })
d := e.newDevice()
dc := e.connectOK(d)
ds := dc.serve(holdHandler(d, release))
defer close(release)
held := func(ip string) int {
n := 0
for i := 0; i < 20; i++ {
if _, err := e.clientFrom(ip, d); err == nil {
n++
}
}
return n
}
if n := held("198.51.100.1"); n != 2 {
t.Fatalf("an ordinary source held %d, cap 2", n)
}
if n := held("192.0.2.1"); n != 8 {
t.Fatalf("an OpenAI address held %d, want 2 x 4", n)
}
seen := map[string]bool{}
for _, o := range ds.openList() {
seen[o.str("peer_ip")+" "+o.str("class")] = true
}
if !seen["192.0.2.1 openai"] || !seen["198.51.100.1 anon"] {
t.Fatalf("OPENs: %v", seen)
}
e.checkTranscript()
_ = fmt.Sprint
}
files/tunnel/outage_test.go 358 lines · 14.9 KB
package main
import (
"strings"
"testing"
"time"
)
// What a node does when its control plane does not answer (the design plan, section 5): what is already authenticated goes on until
// valid_until, a paid device gets 15 minutes more, a trial or a gated device gets none, and nothing new is admitted without a good authorize. Every
// test runs the whole stack (HTTPS client, JWKS cache, feed) against a fake control plane on a clock the test moves by hand.
// pass moves the node's clock forward by d in steps of at most step, giving the node's timers and loops a moment after each.
func (e *testEnv) pass(d, step time.Duration) {
for d > 0 {
s := min(d, step)
e.clk.Advance(s)
time.Sleep(12 * time.Millisecond)
d -= s
}
}
// outageDevice is a device with its connection up, its read loop running and a stream of a client open on it.
type outageDevice struct {
d *testDevice
conn *devConn
ds *devServer
stream *tlsClient
}
func connectWithStream(t *testing.T, e *testEnv, plan devicePlan) *outageDevice {
t.Helper()
d := e.newDeviceNoPlan()
plan.host = d.host
e.cp.set(d.id, &plan)
dc := e.connectOK(d)
ds := dc.serve(nil)
return &outageDevice{d: d, conn: dc, ds: ds, stream: streamOpen(t, e, d)}
}
// validUntil is the moment the OK gave the connection.
func (o *outageDevice) validUntil() time.Time {
return time.Unix(int64(o.conn.ok["body"].(map[string]any)["valid_until"].(float64)), 0)
}
func (o *outageDevice) closed(want string) bool { return containsLine(o.ds.otherLines(), want) }
func (o *outageDevice) renewFails(t *testing.T) {
t.Helper()
if line := o.conn.renew(); line != "ERR control_unavailable" {
t.Fatalf("a RENEW while the control plane is down: %q", line)
}
}
func TestOutage_ADownControlPlaneFor14MinutesKeepsPaidTunnelsAndRefusesNewAttaches(t *testing.T) {
clk := newFakeClock()
e, cp := httpNode(t, clk, nil)
paid := connectWithStream(t, e, devicePlan{validFor: 4 * time.Hour})
cp.setDown(503)
e.pass(14*time.Minute, time.Minute)
// the paid tunnel and its stream are untouched, and the host still serves new clients
if !e.srv.Online(paid.d.host) || paid.stream.dead(50*time.Millisecond) || paid.closed("CLOSE") {
t.Fatal("a paid tunnel did not survive 14 minutes of a down control plane")
}
if c, err := e.clientTLS(paid.d); err != nil {
t.Fatalf("a client of a device that is authenticated: %v", err)
} else {
_ = c.Close()
}
// a NEW attach needs a good authorize: it is refused with a word that says "try later", and nothing was relayed
newcomer := e.newDevice()
conn := e.connect(newcomer)
line := conn.hello(func() string { return e.session(newcomer, nil) }, func(s, n string) string { return e.proof(newcomer, s, n, nil) })
if line != "ERR control_unavailable" {
t.Fatalf("a new attach during the outage: %q", line)
}
if e.srv.Online(newcomer.host) {
t.Fatal("a device that nothing authorized is served")
}
// and the same device that was already attached may RENEW later, when the control plane is back; until then its renewals keep the connection
paid.renewFails(t)
if !e.srv.Online(paid.d.host) {
t.Fatal("a refused RENEW closed the connection")
}
cp.setDown(0)
clk.Advance(2 * time.Second)
if line := e.connect(newcomer).hello(func() string { return e.session(newcomer, nil) }, func(s, n string) string { return e.proof(newcomer, s, n, nil) }); !strings.HasPrefix(line, "OK ") {
t.Fatalf("a new attach once the control plane is back: %q", line)
}
if r := paid.conn.renew(); !strings.HasPrefix(r, "OK ") {
t.Fatalf("renew after the outage: %q", r)
}
}
func TestOutage_ATrialAndAGatedTunnelAreNotExtended(t *testing.T) {
clk := newFakeClock()
e, cp := httpNode(t, clk, nil)
trial := connectWithStream(t, e, devicePlan{plan: "trial", validFor: 15 * time.Minute})
gated := connectWithStream(t, e, devicePlan{status: statusGated, plan: "trial", validFor: 15 * time.Minute})
if gated.conn.ok["body"].(map[string]any)["status"] != "gated" {
t.Fatalf("OK %v", gated.conn.ok)
}
cp.setDown(503)
e.pass(10*time.Minute, time.Minute) // the moment a device renews: valid_until minus 5 minutes
trial.renewFails(t)
gated.renewFails(t)
// gated opens streams like any other device: the card is served by the device
if c, err := e.clientTLS(gated.d); err != nil {
t.Fatalf("a client of a gated device: %v", err)
} else {
_ = c.Close()
}
// five minutes later the grants are over, the control plane still down: no grace for either
e.pass(5*time.Minute+2*time.Second, 30*time.Second)
for name, o := range map[string]*outageDevice{"trial": trial, "gated": gated} {
o := o
eventually(t, name+" CLOSE expired", 3*time.Second, func() bool { return o.closed("CLOSE expired") })
if !o.stream.dead(2 * time.Second) {
t.Errorf("%s: the stream outlived the grant", name)
}
if e.srv.Online(o.d.host) {
t.Errorf("%s: still online", name)
}
}
}
func TestOutage_APaidTunnelOutlivesItsValidUntilByTheGraceAndNotOneSecondMore(t *testing.T) {
if policyData.Authorization.GracePaidS != 900 {
t.Fatalf("the plan says 15 minutes of grace, the policy says %d s", policyData.Authorization.GracePaidS)
}
clk := newFakeClock()
e, cp := httpNode(t, clk, nil)
paid := connectWithStream(t, e, devicePlan{validFor: 15 * time.Minute})
validUntil := paid.validUntil()
cp.setDown(503)
e.pass(10*time.Minute, time.Minute)
paid.renewFails(t)
e.pass(validUntil.Sub(e.now())+time.Second, 30*time.Second)
// past valid_until: the connection and its stream live (grace), but no NEW stream is opened
if paid.closed("CLOSE") || paid.stream.dead(50*time.Millisecond) {
t.Fatal("a paid tunnel was cut at valid_until although the control plane is down")
}
if e.srv.Online(paid.d.host) {
t.Fatal("a stream may be opened after valid_until")
}
if _, err := e.clientTLS(paid.d); err == nil {
t.Fatal("a new client got a stream after valid_until")
}
if !strings.Contains(e.logs.String(), `"msg":"control_grace"`) {
t.Fatal("the grace is not logged")
}
// (the contracts' transcript checker has no notion of the grace, so this test does not run it; the rule it enforces, no OPEN at or after
// valid_until, is the two refusals above)
// just before the grace ends it still lives
e.pass(validUntil.Add(policyDataGrace()).Sub(e.now())-10*time.Second, 30*time.Second)
if paid.closed("CLOSE") || paid.stream.dead(50*time.Millisecond) {
t.Fatal("cut before the grace was over")
}
// and when it is over, the connection and the streams are closed
e.pass(20*time.Second, 5*time.Second)
eventually(t, "CLOSE expired", 3*time.Second, func() bool { return paid.closed("CLOSE expired") })
if !paid.stream.dead(2 * time.Second) {
t.Fatal("the stream outlived the grace")
}
}
func policyDataGrace() time.Duration { return secs(policyData.Authorization.GracePaidS) }
func TestOutage_APaidDeviceTheControlPlaneRefusedGetsNoGrace(t *testing.T) {
clk := newFakeClock()
e, _ := httpNode(t, clk, nil)
// The control plane is UP and says no: a license that ended. The paid plan does not matter, there is nothing to be graceful about.
refused := connectWithStream(t, e, devicePlan{validFor: 15 * time.Minute})
e.pass(10*time.Minute, time.Minute)
e.cp.set(refused.d.id, &devicePlan{host: refused.d.host, err: &ControlError{Status: 402, Code: codeLicenseInactive}})
if r := refused.conn.renew(); r != "ERR license_inactive" {
t.Fatalf("%q", r)
}
e.pass(5*time.Minute+2*time.Second, 30*time.Second)
eventually(t, "CLOSE expired", 3*time.Second, func() bool { return refused.closed("CLOSE expired") })
if !refused.stream.dead(2 * time.Second) {
t.Fatal("the stream outlived the grant of a device the control plane refused")
}
if strings.Contains(e.logs.String(), `"msg":"control_grace"`) {
t.Fatal("a grace was given while the control plane was answering")
}
}
func TestOutage_ADeviceWhoseOwnRenewalHadNoAnswerGetsTheGraceEvenWhileTheRestOfTheControlPlaneWorks(t *testing.T) {
// "subscription_verification_unavailable": the control plane is up but cannot tell whether the subscription is paid. Nothing is granted, and a
// paying device's connection is not cut for it.
clk := newFakeClock()
e, _ := httpNode(t, clk, nil)
o := connectWithStream(t, e, devicePlan{validFor: 15 * time.Minute})
validUntil := o.validUntil()
e.pass(10*time.Minute, time.Minute)
e.cp.set(o.d.id, &devicePlan{host: o.d.host, err: &ControlError{Status: 503, Code: "subscription_verification_unavailable"}})
o.renewFails(t)
e.pass(validUntil.Sub(e.now())+time.Second, 30*time.Second)
if o.closed("CLOSE") || o.stream.dead(50*time.Millisecond) {
t.Fatal("a paying device was cut because the subscription could not be verified")
}
e.pass(policyDataGrace(), time.Minute)
eventually(t, "CLOSE expired", 3*time.Second, func() bool { return o.closed("CLOSE expired") })
}
func TestOutage_ARenewInsideTheGraceEndsItAndTheNewGrantIsTheControlPlanes(t *testing.T) {
clk := newFakeClock()
e, cp := httpNode(t, clk, nil)
o := connectWithStream(t, e, devicePlan{validFor: 15 * time.Minute})
validUntil := o.validUntil()
cp.setDown(503)
e.pass(10*time.Minute, time.Minute)
o.renewFails(t)
e.pass(validUntil.Sub(e.now())+time.Minute, 30*time.Second) // a minute into the grace
if o.closed("CLOSE") {
t.Fatal("closed inside the grace")
}
// the control plane is back: the next RENEW works, and the connection has a new valid_until with no grace attached
cp.setDown(0)
clk.Advance(2 * time.Second)
if r := o.conn.renew(); !strings.HasPrefix(r, "OK ") {
t.Fatalf("renew inside the grace: %q", r)
}
if !e.srv.Online(o.d.host) {
t.Fatal("not served after the renewal")
}
// the old end of the grace passes: nothing happens (the new grant is a minute longer than that)
e.pass(o.validUntil().Add(policyDataGrace()).Sub(e.now())+10*time.Second, 30*time.Second)
if o.closed("CLOSE") || o.stream.dead(50*time.Millisecond) || !e.srv.Online(o.d.host) {
t.Fatal("the old grace closed a connection that was renewed")
}
e.checkTranscript()
}
func TestOutage_APaidTunnelSurvivesATwoHourControlPlaneOutage(t *testing.T) {
clk := newFakeClock()
e, cp := httpNode(t, clk, nil)
paid := connectWithStream(t, e, devicePlan{validFor: 4 * time.Hour})
other := connectWithStream(t, e, devicePlan{validFor: 4 * time.Hour})
validUntil := paid.validUntil()
cp.setDown(503)
e.pass(2*time.Hour, 5*time.Minute)
for name, o := range map[string]*outageDevice{"first": paid, "second": other} {
if !e.srv.Online(o.d.host) || o.closed("CLOSE") || o.stream.dead(50*time.Millisecond) {
t.Fatalf("%s: a paid tunnel did not survive two hours of a down control plane", name)
}
if c, err := e.clientTLS(o.d); err != nil {
t.Fatalf("%s: a new client during the outage: %v", name, err)
} else {
_ = c.Close()
}
}
// by now the node no longer vouches for the control plane's keys (30 minutes), so even a good session is refused as "try later"
newcomer := e.newDevice()
if line := e.connect(newcomer).hello(func() string { return e.session(newcomer, nil) }, func(s, n string) string { return e.proof(newcomer, s, n, nil) }); line != "ERR control_unavailable" {
t.Fatalf("a new attach after two hours of outage: %q", line)
}
// the control plane comes back: the connection renews when it is due, and gets a fresh four hours
cp.setDown(0)
e.pass(validUntil.Add(-5*time.Minute).Sub(e.now()), 20*time.Minute)
if r := paid.conn.renew(); !strings.HasPrefix(r, "OK ") {
t.Fatalf("renew after the outage: %q", r)
}
e.pass(30*time.Minute, 10*time.Minute) // past the old valid_until
if !e.srv.Online(paid.d.host) || paid.closed("CLOSE") {
t.Fatal("the renewed connection was closed at the old valid_until")
}
e.checkTranscript()
}
func TestGated_ATrialThatEndsBecomesGatedAtItsRenewalWithoutAnyEvent(t *testing.T) {
clk := newFakeClock()
e, _ := httpNode(t, clk, nil)
o := connectWithStream(t, e, devicePlan{plan: "trial", validFor: 15 * time.Minute})
if o.conn.ok["body"].(map[string]any)["status"] != "active" {
t.Fatalf("OK %v", o.conn.ok)
}
e.pass(10*time.Minute, time.Minute)
// the trial ended: the control plane now says gated (and never beyond its cutoff)
e.cp.set(o.d.id, &devicePlan{host: o.d.host, status: statusGated, plan: "trial", validFor: 15 * time.Minute})
r := o.conn.renew()
if !strings.HasPrefix(r, "OK ") || !strings.Contains(r, `"status":"gated"`) {
t.Fatalf("renew of a trial that ended: %q", r)
}
if c, err := e.clientTLS(o.d); err != nil {
t.Fatalf("a gated device serves streams (the card is the device's): %v", err)
} else {
_ = c.Close()
}
// and it is not extended by anything the node thinks: with the control plane down at the next end, it closes
e.cpDownThrough(15 * time.Minute)
eventually(t, "CLOSE expired", 3*time.Second, func() bool { return o.closed("CLOSE expired") })
e.checkTranscript()
}
// cpDownThrough takes the control plane down (every call of the node answers 503) and lets d pass.
func (e *testEnv) cpDownThrough(d time.Duration) {
e.cp.mu.Lock()
for _, p := range e.cp.plans {
p.err = &ControlError{Status: 503, Code: "end_to_end_unavailable"}
}
e.cp.mu.Unlock()
e.pass(d+2*time.Second, time.Minute)
}
func TestOutage_APaidDeviceTheControlPlaneRefusedGetsNoGraceWhenTheControlPlaneGoesDownLater(t *testing.T) {
clk := newFakeClock()
e, cp := httpNode(t, clk, nil)
o := connectWithStream(t, e, devicePlan{validFor: 15 * time.Minute})
validUntil := o.validUntil()
e.pass(2*time.Minute, time.Minute)
e.cp.set(o.d.id, &devicePlan{host: o.d.host, err: &ControlError{Status: 402, Code: codeLicenseInactive}})
if r := o.conn.renew(); r != "ERR license_inactive" {
t.Fatalf("%q", r)
}
// the whole control plane goes down afterwards: the node cannot tell one device from another, but it has been told no for this one
cp.setDown(503)
e.pass(validUntil.Sub(e.now())+2*time.Second, time.Minute)
eventually(t, "CLOSE expired", 3*time.Second, func() bool { return o.closed("CLOSE expired") })
if strings.Contains(e.logs.String(), `"msg":"control_grace"`) {
t.Fatal("a device the control plane refused was given the grace")
}
}
func TestOutage_ARenewTheNodeItselfCouldNotSendIsNotTheControlPlaneBeingDown(t *testing.T) {
// The node's own queue for authorize is full: the control plane was never asked, and answers everyone else. The device gets no grace for it.
clk := newFakeClock()
e, _ := httpNode(t, clk, func(c *Config) { c.MaxUnauth = 1 })
o := connectWithStream(t, e, devicePlan{validFor: 15 * time.Minute})
validUntil := o.validUntil()
e.pass(2*time.Minute, time.Minute)
slow := e.newDevice()
e.cp.set(slow.id, &devicePlan{host: slow.host, validFor: time.Hour, delay: 2 * time.Second})
hello := e.helloAsync(slow)
eventually(t, "the other device's authorize in flight", 3*time.Second, func() bool { return e.cp.callsFor(slow.id, "hello") == 1 })
o.renewFails(t)
if n := e.cp.callsFor(o.d.id, "renew"); n != 0 {
t.Fatalf("the control plane was asked %d times: the queue was not full", n)
}
<-hello
e.pass(validUntil.Sub(e.now())+2*time.Second, time.Minute)
eventually(t, "CLOSE expired", 3*time.Second, func() bool { return o.closed("CLOSE expired") })
if strings.Contains(e.logs.String(), `"msg":"control_grace"`) {
t.Fatal("a RENEW that never left the node was taken for a control plane that does not answer")
}
}
files/tunnel/policy.go 159 lines · 6.5 KB
package main
import (
_ "embed"
"encoding/json"
"fmt"
"regexp"
"strings"
"time"
)
// policy.json is the byte-identical copy of contracts/policy.json that `the contract generator's sync` writes (the Go module cannot read
// contracts/ at runtime). Every number, window and pattern of the node comes from it: nothing here types one again.
//
//go:embed policy.json
var policyJSON []byte
// Policy is the part of the ALL 2.0 policy table the node reads.
type Policy struct {
Protocol struct {
Version int `json:"version"`
ALPN string `json:"alpn"`
LineMaxBytes int `json:"line_max_bytes"`
OpenLineMaxBytes int `json:"open_line_max_bytes"`
StatementLineMaxBytes int `json:"statement_line_max_bytes"`
JWTMaxBytes int `json:"jwt_max_bytes"`
FirstLineTimeoutS int `json:"first_line_timeout_s"`
AttachTimeoutS int `json:"attach_timeout_s"`
HintMaxAddresses int `json:"hint_max_addresses"`
ProofHTM string `json:"proof_htm"`
ProofHTUPath string `json:"proof_htu_path"`
SourceClasses []string `json:"source_classes"`
CloseReasons []string `json:"close_reasons"`
ErrorCodes []string `json:"error_codes"`
} `json:"protocol"`
Timing struct {
PingIntervalS int `json:"ping_interval_s"`
DeadPeerS int `json:"dead_peer_s"`
StreamIdleS int `json:"stream_idle_s"`
StreamMaxAgeS int `json:"stream_max_age_s"`
HalfCloseS int `json:"half_close_s"`
BackoffMinS int `json:"backoff_min_s"`
RenewBeforeValidUntilS int `json:"renew_before_valid_until_s"`
BackoffMaxS int `json:"backoff_max_s"`
DrainQuietS int `json:"drain_quiet_s"`
FeedPollS int `json:"feed_poll_s"`
FeedApplyDeadlineS int `json:"feed_apply_deadline_s"`
} `json:"timing"`
Authorization struct {
SessionTTLS int `json:"session_ttl_s"`
ProofTTLS int `json:"proof_ttl_s"`
NodeSkewS int `json:"node_skew_s"`
NodeSkewMaxS int `json:"node_skew_max_s"`
AuthorizeMaxConcur int `json:"authorize_max_concurrent"`
NodeAssertionTTLS int `json:"node_assertion_ttl_s"`
GracePaidS int `json:"grace_paid_s"`
} `json:"authorization"`
Identity struct {
Issuer string `json:"issuer"`
AudTunnel string `json:"aud_tunnel"`
AudNodeAssert string `json:"aud_node_assertion"`
FQDNRegex string `json:"fqdn_regex"`
FQDNMaxLength int `json:"fqdn_max_length"`
LabelRegex string `json:"label_regex"`
DeviceIDRegex string `json:"device_id_regex"`
StreamIDRegex string `json:"stream_id_regex"`
AttachSecretRE string `json:"attach_secret_regex"`
NonceRegex string `json:"nonce_regex"`
JTIRegex string `json:"jti_regex"`
KIDRegex string `json:"kid_regex"`
} `json:"identity"`
JWKS struct {
CacheS int `json:"cache_s"`
MaxKeys int `json:"max_keys"`
StaleFailClosedS int `json:"stale_fail_closed_s"`
} `json:"jwks"`
Enums struct {
FeedEventType []string `json:"feed_event_type"`
} `json:"enums"`
NodeLimits struct {
ClientHelloDeadlineS int `json:"client_hello_deadline_s"`
PerIPConcurrent int `json:"per_ip_concurrent"`
PerIPNewPerS int `json:"per_ip_new_per_s"`
IPv6PrefixBits int `json:"ipv6_prefix_bits"`
LabelStreamsMax int `json:"label_streams_max"`
LabelAnonSlice int `json:"label_anon_slice"`
LabelHintSlice int `json:"label_hint_slice"`
HintTTLS int `json:"hint_ttl_s"`
GlobalStreamsMax int `json:"global_streams_max"`
OpenAIReservedPct int `json:"openai_reserved_pct"`
OpenAIPrefixRefreshS int `json:"openai_prefix_refresh_s"`
OpenAIPrefixMax int `json:"openai_prefix_max"`
OpenAIPrefixMinV4Bits int `json:"openai_prefix_min_v4_bits"`
OpenAIPrefixMinV6Bits int `json:"openai_prefix_min_v6_bits"`
OpenAIPrefixBodyMaxBytes int `json:"openai_prefix_body_max_bytes"`
OpenAIPerIPFactor int `json:"openai_per_ip_factor"`
DevicePerIPFactor int `json:"device_per_ip_factor"`
IPTableMax int `json:"ip_table_max"`
UnknownSNIPerIP int `json:"unknown_sni_per_ip"`
UnknownSNIWindowS int `json:"unknown_sni_window_s"`
UnknownSNIBanS int `json:"unknown_sni_ban_s"`
LabelNewPerS int `json:"label_new_per_s"`
LabelNewBurst int `json:"label_new_burst"`
HintAddressesPerLabel int `json:"hint_addresses_per_label"`
TCPKeepaliveS int `json:"tcp_keepalive_s"`
MaxUnauthenticatedControl int `json:"max_unauthenticated_control"`
} `json:"node_limits"`
Gated struct {
BytesPerDay int64 `json:"bytes_per_day"`
StreamsPerDay int `json:"streams_per_day"`
} `json:"gated"`
}
// policyData is the parsed table, read once. The file is embedded, so a bad one is a build error found by the first test, never a runtime state.
var policyData = func() *Policy {
var p Policy
if err := json.Unmarshal(policyJSON, &p); err != nil {
panic("tunnel/policy.json: " + err.Error())
}
return &p
}()
// secs turns a policy value in seconds into a Duration.
func secs(n int) time.Duration { return time.Duration(n) * time.Second }
// anchored compiles one of the policy's ^...$ patterns (RE2: `$` is the end of the text, so a trailing newline never matches).
func anchored(pattern string) *regexp.Regexp {
if !strings.HasPrefix(pattern, "^") || !strings.HasSuffix(pattern, "$") {
panic(fmt.Sprintf("policy patterns are anchored: %s", pattern))
}
return regexp.MustCompile(pattern)
}
// The patterns of the contracts, compiled from the policy and never written again.
var (
reLabel = anchored(policyData.Identity.LabelRegex)
reFQDN = anchored(policyData.Identity.FQDNRegex)
reDeviceID = anchored(policyData.Identity.DeviceIDRegex)
reStreamID = anchored(policyData.Identity.StreamIDRegex)
reSecret = anchored(policyData.Identity.AttachSecretRE)
reNonce = anchored(policyData.Identity.NonceRegex)
reJTI = anchored(policyData.Identity.JTIRegex)
reKID = anchored(policyData.Identity.KIDRegex)
)
// inList reports whether s is one of the words.
func inList(words []string, s string) bool {
for _, w := range words {
if w == s {
return true
}
}
return false
}
// validFQDN is the contracts' device host: exactly one label under a zone, at most the policy length.
func validFQDN(s string) bool {
return len(s) <= policyData.Identity.FQDNMaxLength && reFQDN.MatchString(s)
}
files/tunnel/policy.json 229 lines · 6.0 KB
{
"$comment": "ALL 2.0 policy table: the ONE place where a number, a window or a limit of the end-to-end path is written down (plan section 0 and 7.1). The tunnel node (Go), the control plane (TypeScript) and the device (Python) read it; nobody copies a value into code (the contract generator lint fails the suite when one does). Units are in the key suffix: _s seconds, _bytes, _days, _pct percent. Keys listed in `provisional` are the plan's initial values: they are retuned after G0 and G1 and ONLY here. This is the node projection (the `copies` of the contract lint table): only the sections that runtime reads.",
"version": 1,
"provisional": [
"timing.stream_idle_s",
"timing.stream_max_age_s",
"timing.ping_interval_s",
"timing.dead_peer_s",
"node_limits.per_ip_concurrent",
"node_limits.per_ip_new_per_s",
"node_limits.label_streams_max",
"node_limits.label_anon_slice",
"node_limits.label_hint_slice",
"node_limits.global_streams_max",
"node_limits.openai_reserved_pct",
"node_limits.openai_per_ip_factor",
"node_limits.device_per_ip_factor",
"node_limits.ip_table_max",
"node_limits.unknown_sni_per_ip",
"node_limits.unknown_sni_window_s",
"node_limits.unknown_sni_ban_s",
"node_limits.label_new_per_s",
"node_limits.label_new_burst",
"node_limits.hint_addresses_per_label"
],
"protocol": {
"version": 2,
"alpn": "all-tunnel/2",
"line_max_bytes": 8192,
"open_line_max_bytes": 192,
"statement_line_max_bytes": 65552,
"jwt_max_bytes": 4096,
"first_line_timeout_s": 10,
"attach_timeout_s": 10,
"stream_id_bytes": 16,
"attach_secret_bytes": 32,
"nonce_bytes": 16,
"hint_max_addresses": 16,
"proof_htm": "GET",
"proof_htu_path": "/all-tunnel/2",
"node_assertion_header": "x-all-node-assertion",
"source_classes": [
"openai",
"hint",
"anon"
],
"close_reasons": [
"replaced",
"revoked",
"expired",
"released",
"killed",
"paused",
"drain",
"idle",
"protocol",
"shutdown"
],
"error_codes": [
"bad_hello",
"bad_command",
"unsupported_version",
"bad_proof",
"auth",
"clock_skew",
"unknown_stream",
"bad_attach",
"license_inactive",
"device_revoked",
"paused",
"busy",
"control_unavailable",
"line_too_long"
]
},
"timing": {
"ping_interval_s": 20,
"dead_peer_s": 45,
"stream_idle_s": 1800,
"stream_max_age_s": 14400,
"half_close_s": 30,
"backoff_min_s": 1,
"backoff_max_s": 60,
"renew_before_valid_until_s": 300,
"drain_quiet_s": 1200,
"feed_poll_s": 15,
"feed_apply_deadline_s": 20,
"reconnect_after_wake_s": 5
},
"authorization": {
"valid_until_paid_s": 14400,
"valid_until_trial_s": 900,
"valid_until_gated_s": 900,
"grace_paid_s": 900,
"authorize_max_concurrent": 50,
"session_ttl_s": 300,
"proof_ttl_s": 60,
"node_assertion_ttl_s": 60,
"node_skew_s": 5,
"node_skew_max_s": 30,
"access_ttl_paid_s": 14400,
"access_ttl_trial_s": 600,
"access_ttl_paywall_s": 600,
"access_future_slack_s": 60,
"stripe_status_cache_s": 120,
"last_seen_write_interval_s": 300
},
"gated": {
"bytes_per_day": 10485760,
"streams_per_day": 200
},
"identity": {
"issuer": "https://api.all-mcp.fyi",
"aud_tunnel": "all-tunnel",
"aud_access": "all-device-access",
"aud_node_assertion": "all-tunnel-control",
"label_length": 20,
"label_alphabet": "abcdefghijklmnopqrstuvwxyz234567",
"label_regex": "^[a-z2-7]{20}$",
"fqdn_regex": "^[a-z2-7]{20}\\.(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\\.)+[a-z]{2,63}$",
"fqdn_max_length": 253,
"tombstone_idle_months": 12,
"device_id_regex": "^[a-f0-9]{32}$",
"stream_id_regex": "^[0-9a-f]{32}$",
"attach_secret_regex": "^[A-Za-z0-9_-]{43}$",
"nonce_regex": "^[A-Za-z0-9_-]{22,64}$",
"jti_regex": "^[A-Za-z0-9_-]{16,128}$",
"kid_regex": "^[A-Za-z0-9._-]{1,128}$"
},
"jwks": {
"cache_s": 300,
"max_keys": 16,
"stale_fail_closed_s": 1800
},
"node_limits": {
"client_hello_deadline_s": 5,
"per_ip_concurrent": 16,
"per_ip_new_per_s": 10,
"ipv6_prefix_bits": 64,
"label_streams_max": 64,
"label_anon_slice": 16,
"label_hint_slice": 48,
"hint_ttl_s": 900,
"global_streams_max": 2048,
"openai_reserved_pct": 25,
"openai_prefix_refresh_s": 86400,
"openai_prefix_max": 2048,
"openai_prefix_min_v4_bits": 16,
"openai_prefix_min_v6_bits": 32,
"openai_prefix_body_max_bytes": 1048576,
"openai_per_ip_factor": 8,
"device_per_ip_factor": 8,
"ip_table_max": 65536,
"unknown_sni_per_ip": 20,
"unknown_sni_window_s": 60,
"unknown_sni_ban_s": 600,
"label_new_per_s": 20,
"label_new_burst": 32,
"hint_addresses_per_label": 64,
"tcp_keepalive_s": 30,
"max_unauthenticated_control": 256
},
"enums": {
"authorize_status": [
"active",
"gated",
"paused"
],
"statement_mode": [
"active",
"paywall"
],
"budget_class": [
"paid",
"trial",
"flex"
],
"acme_purpose": [
"first",
"renewal"
],
"reserve_status": [
"granted",
"deferred"
],
"feed_event_type": [
"revoke",
"pause",
"resume",
"license_end",
"release",
"kill",
"kill_clear"
],
"e2e_state": [
"off",
"unenrolled",
"waiting",
"issuing",
"connecting",
"ready",
"degraded",
"clock_skew",
"compat_only",
"killed"
],
"e2e_cohort": [
"paid",
"trial",
"flex",
"legacy"
],
"ineligible_reason": [
"update_not_in_plan",
"legacy_credential",
"unsupported_version"
],
"selfcheck_reason": [
"chain",
"expired",
"spki",
"name",
"tls_version",
"healthz",
"unreachable",
"timeout"
]
}
}
files/tunnel/proto.go 426 lines · 13.2 KB
package main
import (
"bufio"
"errors"
"net/netip"
"strconv"
"strings"
"time"
)
// Protocol v2 on the wire (the contract, section 2, contracts/schemas/wire-v2.grammar.json). A line is ASCII 0x20-0x7e ended by LF, fields
// are separated by exactly one space and none is empty. Two kinds of connection are told apart by the first line: a control connection (HELLO) and a
// data connection (ATTACH, one per client stream).
// Error words of ERR lines (policy protocol.error_codes) and the words of the node's own decisions.
const (
codeBadHello = "bad_hello"
codeBadCommand = "bad_command"
codeUnsupportedVersion = "unsupported_version"
codeBadProof = "bad_proof"
codeAuth = "auth"
codeClockSkew = "clock_skew"
codeUnknownStream = "unknown_stream"
codeBadAttach = "bad_attach"
codeLicenseInactive = "license_inactive"
codeDeviceRevoked = "device_revoked"
codePaused = "paused"
codeBusy = "busy"
codeControlUnavailable = "control_unavailable"
codeLineTooLong = "line_too_long"
)
// Reasons of a CLOSE line (policy protocol.close_reasons) the node sends.
const (
closeReplaced = "replaced"
closeRevoked = "revoked"
closeExpired = "expired"
closeIdle = "idle"
closeProtocol = "protocol"
closeShutdown = "shutdown"
closeReleased = "released"
closeKilled = "killed"
closeDrain = "drain"
closePaused = "paused"
)
// Classes of the source of a client stream (policy protocol.source_classes).
const (
classOpenAI = "openai"
classHint = "hint"
classAnon = "anon"
)
// Direction of a line: d2n is device to node, n2d is node to device, both is CLOSE.
const (
dirD2N = "d2n"
dirN2D = "n2d"
dirBoth = "both"
)
// protoError is a refused line: code is the word of the ERR the node answers with.
type protoError struct{ code string }
func (e *protoError) Error() string { return e.code }
func protoFail(code string) error { return &protoError{code} }
// errCode is the ERR word of an error: its own when it is a protoError, line_too_long for an over-long line, bad_command otherwise.
func errCode(err error) string {
var pe *protoError
if errors.As(err, &pe) {
return pe.code
}
if errors.Is(err, bufio.ErrBufferFull) {
return codeLineTooLong
}
return codeBadCommand
}
// message is one parsed line: {"verb": ..., <field>: ...}. HINT gives "addresses" ([]any of strings) and OK gives "body".
type message map[string]any
func (m message) str(key string) string { s, _ := m[key].(string); return s }
type fieldSpec struct {
name, kind string
err string // ERR word when the field is wrong (default bad_command)
}
type verbSpec struct {
dir string
max int // longest line, LF included; 0 = protocol.line_max_bytes
fields []fieldSpec
optional []fieldSpec
repeat *fieldSpec // HINT: one to hint_max_addresses of the same kind
json bool // OK: the rest of the line is one canonical JSON object
}
// verbs is wire-v2.grammar.json, written out. protocol_test.go runs every vector of contracts/vectors/lines.json through it.
var verbs = map[string]verbSpec{
"HELLO": {dir: dirD2N, fields: []fieldSpec{{"version", "version", ""}, {"session", "jwt", codeBadHello}}},
"PROOF": {dir: dirD2N, fields: []fieldSpec{{"proof", "jwt", codeBadProof}}},
"RENEW": {dir: dirD2N, fields: []fieldSpec{{"session", "jwt", codeBadHello}}},
"PING": {dir: dirD2N},
"HINT": {dir: dirD2N, repeat: &fieldSpec{"address", "ip", ""}},
"CLOSE": {dir: dirBoth, optional: []fieldSpec{{"reason", "close_reason", ""}}},
"ATTACH": {dir: dirD2N, fields: []fieldSpec{
{"stream_id", "stream_id", codeBadAttach},
{"secret", "attach_secret", codeBadAttach},
}},
"CHALLENGE": {dir: dirN2D, fields: []fieldSpec{{"nonce", "nonce", ""}}},
"OK": {dir: dirN2D, json: true},
"ERR": {dir: dirN2D, fields: []fieldSpec{{"code", "error_code", ""}}},
"OPEN": {dir: dirN2D, max: policyData.Protocol.OpenLineMaxBytes, fields: []fieldSpec{
{"stream_id", "stream_id", ""},
{"secret", "attach_secret", ""},
{"peer_ip", "ip", ""},
{"class", "source_class", ""},
}},
"PONG": {dir: dirN2D},
"STATEMENT": {dir: dirN2D, max: policyData.Protocol.StatementLineMaxBytes, fields: []fieldSpec{{"statement", "statement", ""}}},
"DRAIN": {dir: dirN2D, fields: []fieldSpec{{"quiet_s", "seconds", ""}}},
}
// isJWSShape is the textual shape of a compact JWS: three non-empty base64url segments.
func isJWSShape(s string) bool {
dots := 0
segment := 0
for i := 0; i < len(s); i++ {
c := s[i]
switch {
case c == '.':
if segment == 0 {
return false
}
dots++
segment = 0
case c >= 'A' && c <= 'Z', c >= 'a' && c <= 'z', c >= '0' && c <= '9', c == '-', c == '_':
segment++
default:
return false
}
}
return dots == 2 && segment > 0
}
// validIP is the contracts' address: IPv4 dotted quad without leading zeros or RFC 4291 text in lower case, no zone, no brackets.
func validIP(s string) bool {
if s == "" || s != strings.ToLower(s) || strings.ContainsAny(s, "%[] \t\r\n/") {
return false
}
_, err := netip.ParseAddr(s)
return err == nil
}
func fieldOK(kind, token string) bool {
switch kind {
case "version":
return len(token) >= 1 && len(token) <= 3 && allDigits(token)
case "jwt":
return len(token) <= policyData.Protocol.JWTMaxBytes && isJWSShape(token)
case "statement":
return isJWSShape(token)
case "nonce":
return reNonce.MatchString(token)
case "stream_id":
return reStreamID.MatchString(token)
case "attach_secret":
return reSecret.MatchString(token)
case "ip":
return validIP(token)
case "source_class":
return inList(policyData.Protocol.SourceClasses, token)
case "close_reason":
return inList(policyData.Protocol.CloseReasons, token)
case "error_code":
return inList(policyData.Protocol.ErrorCodes, token)
case "seconds":
return token == "0" || (len(token) <= 6 && token[0] != '0' && allDigits(token))
}
return false
}
func allDigits(s string) bool {
for i := 0; i < len(s); i++ {
if s[i] < '0' || s[i] > '9' {
return false
}
}
return s != ""
}
// parseLine parses one protocol line (the text before the LF) travelling in dir ("d2n" or "n2d"). The error is a *protoError whose code is the word
// the node answers with.
func parseLine(line, dir string) (message, error) {
for i := 0; i < len(line); i++ {
if line[i] < 0x20 || line[i] > 0x7e {
return nil, protoFail(codeBadCommand)
}
}
verb, rest, hasSpace := strings.Cut(line, " ")
spec, known := verbs[verb]
limit := policyData.Protocol.LineMaxBytes
if known && spec.max > 0 {
limit = spec.max
}
if len(line)+1 > limit {
return nil, protoFail(codeLineTooLong)
}
if !known || (spec.dir != dir && spec.dir != dirBoth) || line == "" || (hasSpace && rest == "") {
return nil, protoFail(codeBadCommand)
}
out := message{"verb": verb}
if spec.json {
if !hasSpace {
return nil, protoFail(codeBadCommand)
}
body, err := parseJSON(rest)
if err != nil || checkOKBody(body) != nil {
return nil, protoFail(codeBadCommand)
}
out["body"] = body
return out, nil
}
var tokens []string
if hasSpace {
tokens = strings.Split(rest, " ")
}
for _, token := range tokens {
if token == "" {
return nil, protoFail(codeBadCommand)
}
}
if spec.repeat != nil {
if len(tokens) < 1 || len(tokens) > policyData.Protocol.HintMaxAddresses {
return nil, protoFail(codeBadCommand)
}
addresses := make([]any, len(tokens))
for i, token := range tokens {
if !fieldOK(spec.repeat.kind, token) {
return nil, protoFail(codeBadCommand)
}
addresses[i] = token
}
out["addresses"] = addresses
return out, nil
}
rules := append(append([]fieldSpec{}, spec.fields...), spec.optional...)
if len(tokens) < len(spec.fields) || len(tokens) > len(rules) {
return nil, protoFail(codeBadCommand)
}
for i, token := range tokens {
rule := rules[i]
if verb == "HELLO" && rule.name == "version" {
// the protocol version is 2: another number is unsupported_version, anything else is a malformed line
if token != strconv.Itoa(policyData.Protocol.Version) {
if fieldOK(rule.kind, token) {
return nil, protoFail(codeUnsupportedVersion)
}
return nil, protoFail(codeBadCommand)
}
} else if !fieldOK(rule.kind, token) {
code := rule.err
if code == "" {
code = codeBadCommand
}
return nil, protoFail(code)
}
out[rule.name] = token
}
return out, nil
}
// formatLine is the canonical line (without LF) of a message the node sends; it parses its own output, so a node can never write a line its
// own grammar refuses.
func formatLine(m message) string {
verb := m.str("verb")
spec := verbs[verb]
var line string
switch {
case spec.json:
line = verb + " " + canonicalJSON(m["body"])
case spec.repeat != nil:
parts := []string{verb}
for _, address := range m["addresses"].([]any) {
parts = append(parts, address.(string))
}
line = strings.Join(parts, " ")
default:
parts := []string{verb}
for _, rule := range append(append([]fieldSpec{}, spec.fields...), spec.optional...) {
if v, ok := m[rule.name]; ok {
parts = append(parts, v.(string))
}
}
line = strings.Join(parts, " ")
}
dir := spec.dir
if dir == dirBoth {
dir = dirD2N
}
if _, err := parseLine(line, dir); err != nil {
panic("tunnel: formatted a line its own grammar refuses: " + verb + ": " + err.Error())
}
return line
}
// statementLine is the STATEMENT the node forwards, or "" when the statement is not one the grammar allows (too long, not a JWS): it is then dropped,
// never written, never a panic. The statement itself is the control plane's signed access statement; the node passes it on as it is.
func statementLine(statement string) string {
line := "STATEMENT " + statement
if _, err := parseLine(line, dirN2D); err != nil {
return ""
}
return line
}
// drainLine tells a device the node is going down and how long it keeps its open streams.
func drainLine(quiet time.Duration) string {
return formatLine(message{"verb": "DRAIN", "quiet_s": strconv.Itoa(int(quiet / time.Second))})
}
func errLine(code string) string { return formatLine(message{"verb": "ERR", "code": code}) }
func closeLine(reason string) string {
if reason == "" {
return "CLOSE"
}
return formatLine(message{"verb": "CLOSE", "reason": reason})
}
// okBody is what an OK answers: the control plane's decision, relayed (the contract, section 2). Host, status, valid_until and
// server_time are what the control plane answered; policy is that answer's windows, bounded by policy.json.
type okBody struct {
Node string
Host string
Status string
ValidUntil int64
ServerTime int64
Policy okPolicy
}
type okPolicy struct {
PingIntervalS, DeadPeerS, StreamIdleS, StreamMaxAgeS, RenewBeforeValidUntilS int64
}
func (p okPolicy) toMap() map[string]any {
return map[string]any{"ping_interval_s": p.PingIntervalS, "dead_peer_s": p.DeadPeerS, "stream_idle_s": p.StreamIdleS,
"stream_max_age_s": p.StreamMaxAgeS, "renew_before_valid_until_s": p.RenewBeforeValidUntilS}
}
func (b okBody) line() string {
return formatLine(message{"verb": "OK", "body": map[string]any{"v": policyData.Protocol.Version, "node": b.Node, "host": b.Host, "status": b.Status,
"valid_until": b.ValidUntil, "server_time": b.ServerTime, "policy": b.Policy.toMap()}})
}
// epochOK is the contracts' epoch in seconds.
func epochOK(v any) (int64, bool) {
f, ok := v.(float64)
if !ok || f != float64(int64(f)) || f < 1 || f > epochMax {
return 0, false
}
return int64(f), true
}
const epochMax = 4102444800 // 2100-01-01, common.schema.json epoch_s
var okPolicyKeys = []string{"ping_interval_s", "dead_peer_s", "stream_idle_s", "stream_max_age_s", "renew_before_valid_until_s"}
// checkPolicyBlock validates the five windows of an authorize answer or an OK (integers 1..86400: capped at a day only to refuse nonsense).
func checkPolicyBlock(v any) (okPolicy, bool) {
m, ok := v.(map[string]any)
if !ok || len(m) != len(okPolicyKeys) {
return okPolicy{}, false
}
var got [5]int64
for i, key := range okPolicyKeys {
f, ok := m[key].(float64)
if !ok || f != float64(int64(f)) || f < 1 || f > 86400 {
return okPolicy{}, false
}
got[i] = int64(f)
}
return okPolicy{got[0], got[1], got[2], got[3], got[4]}, true
}
var reNodeID = anchored(`^[a-z0-9]{1,16}$`)
// checkOKBody validates ok-body.schema.json.
func checkOKBody(v any) error {
m, ok := v.(map[string]any)
if !ok || len(m) != 7 {
return errors.New("ok body shape")
}
if f, ok := m["v"].(float64); !ok || f != float64(policyData.Protocol.Version) {
return errors.New("ok body v")
}
if n, ok := m["node"].(string); !ok || !reNodeID.MatchString(n) {
return errors.New("ok body node")
}
if h, ok := m["host"].(string); !ok || !validFQDN(h) {
return errors.New("ok body host")
}
if s, ok := m["status"].(string); !ok || !validStatus(s) {
return errors.New("ok body status")
}
for _, key := range []string{"valid_until", "server_time"} {
if _, ok := epochOK(m[key]); !ok {
return errors.New("ok body " + key)
}
}
if _, ok := checkPolicyBlock(m["policy"]); !ok {
return errors.New("ok body policy")
}
return nil
}
// Statuses of an authorize answer (policy enums.authorize_status).
const (
statusActive = "active"
statusGated = "gated"
statusPaused = "paused"
)
func validStatus(s string) bool { return s == statusActive || s == statusGated || s == statusPaused }
files/tunnel/protocol_test.go 343 lines · 13.5 KB
package main
// ALL 2.0, WP05: the node's own protocol code (policy.go, proto.go, auth.go, jwks.go) against the contracts' golden vectors. The Go reference verifier of
// WP01 (contracts_verify_test.go) and the node are two independent implementations; each vector must get the same verdict and the same error word from
// both. Vectors "nobody has seen" run the same way: ALL_CONTRACT_VECTORS_DIR points ctVectors at a set made with another salt.
import (
"encoding/json"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
)
// tunnel/policy.json is the node's PROJECTION of contracts/policy.json (the contract lint table `copies`: the contract generator's sync writes it): only the sections the node
// reads, each one as it is in the table. So the test compares section by section, and the sections the node's code reads must all be there.
func TestPolicyCopyIsTheNodesProjectionOfTheContracts(t *testing.T) {
want, err := os.ReadFile(filepath.Join(ctContracts, "policy.json"))
if err != nil {
t.Fatal(err)
}
var full, mine map[string]any
if err := json.Unmarshal(want, &full); err != nil {
t.Fatal(err)
}
if err := json.Unmarshal(policyJSON, &mine); err != nil {
t.Fatal(err)
}
for _, section := range []string{"protocol", "timing", "node_limits", "authorization", "gated", "identity", "jwks", "enums"} {
if _, ok := mine[section]; !ok {
t.Errorf("tunnel/policy.json has no %q section: the node reads it (the contract lint table `copies`, then the contract generator's sync)", section)
}
}
for name, section := range mine {
if name == "$comment" || name == "provisional" {
continue
}
whole, ok := full[name]
if !ok {
t.Errorf("section %q of tunnel/policy.json is not in contracts/policy.json", name)
continue
}
mineMap, mineIsMap := section.(map[string]any)
wholeMap, _ := whole.(map[string]any)
if !mineIsMap {
if !reflect.DeepEqual(section, whole) {
t.Errorf("%s of tunnel/policy.json differs from contracts/policy.json: run the contract generator's sync", name)
}
continue
}
for key, value := range mineMap {
if !reflect.DeepEqual(value, wholeMap[key]) {
t.Errorf("%s.%s of tunnel/policy.json differs from contracts/policy.json: run the contract generator's sync", name, key)
}
}
}
}
func TestPolicyDefaultsOfTheNodeAreThePolicysNumbers(t *testing.T) {
cfg := DefaultConfig()
for name, got := range map[string]time.Duration{
"HelloTimeout": cfg.HelloTimeout, "AttachTimeout": cfg.AttachTimeout, "FirstLineTimeout": cfg.FirstLineTimeout, "DeadPeer": cfg.DeadPeer,
"StreamIdle": cfg.StreamIdle, "StreamMaxAge": cfg.StreamMaxAge, "HalfCloseTimeout": cfg.HalfCloseTimeout, "ClockSkew": cfg.ClockSkew, "MinRenewGap": cfg.MinRenewGap,
} {
want := map[string]string{
"HelloTimeout": "node_limits.client_hello_deadline_s", "AttachTimeout": "protocol.attach_timeout_s", "FirstLineTimeout": "protocol.first_line_timeout_s",
"DeadPeer": "timing.dead_peer_s", "StreamIdle": "timing.stream_idle_s", "StreamMaxAge": "timing.stream_max_age_s",
"HalfCloseTimeout": "timing.half_close_s", "ClockSkew": "authorization.node_skew_s", "MinRenewGap": "timing.backoff_min_s",
}[name]
if got != time.Duration(ctPolInt(want))*time.Second {
t.Errorf("%s = %s, policy %s = %d", name, got, want, ctPolInt(want))
}
}
if cfg.MaxStreams != ctPolInt("node_limits.global_streams_max") || cfg.MaxStreamsPerHost != ctPolInt("node_limits.label_streams_max") ||
cfg.MaxUnauth != ctPolInt("node_limits.max_unauthenticated_control") || cfg.AuthorizeMax != ctPolInt("authorization.authorize_max_concurrent") {
t.Errorf("limits %d %d %d %d", cfg.MaxStreams, cfg.MaxStreamsPerHost, cfg.MaxUnauth, cfg.AuthorizeMax)
}
}
func TestWireVectorsGetTheSameVerdictFromTheNode(t *testing.T) {
for _, v := range ctList(t, ctVectors(t, "lines.json"), "lines") {
line, direction := v["line"].(string), v["dir"].(string)
parsed, err := parseLine(line, direction)
if v["ok"] == true {
if err != nil {
t.Errorf("%v: %v", v["id"], err)
continue
}
if !ctEqual(map[string]any(parsed), v["parsed"]) {
t.Errorf("%v: parsed %v, want %v", v["id"], parsed, v["parsed"])
}
if v["canonical"] == true && formatLine(parsed) != line {
t.Errorf("%v: canonical form %q, want %q", v["id"], formatLine(parsed), line)
}
continue
}
if err == nil {
t.Errorf("%v: accepted %q", v["id"], truncate(line))
continue
}
if code := errCode(err); code != v["error"] {
t.Errorf("%v: error %q, want %q", v["id"], code, v["error"])
}
for _, suffix := range []string{"\n", "\x00", "\r"} {
if _, err := parseLine(line+suffix, direction); err == nil {
t.Errorf("%v: accepted with suffix %q", v["id"], suffix)
}
}
}
}
func TestAddressVectorsGetTheSameVerdictFromTheNode(t *testing.T) {
for _, v := range ctList(t, ctVectors(t, "lines.json"), "ips") {
if got := validIP(v["value"].(string)); got != (v["ok"] == true) {
t.Errorf("%v: %q gave %v", v["id"], v["value"], got)
}
}
}
func TestLabelAndHostVectorsGetTheSameVerdictFromTheNode(t *testing.T) {
doc := ctVectors(t, "labels.json")
for _, v := range ctList(t, doc, "labels") {
if got := reLabel.MatchString(v["value"].(string)); got != (v["ok"] == true) {
t.Errorf("%v: label %q gave %v", v["id"], v["value"], got)
}
}
for _, v := range ctList(t, doc, "fqdns") {
if got := validFQDN(v["value"].(string)); got != (v["ok"] == true) && v["zones"] == nil {
t.Errorf("%v: host %q gave %v", v["id"], v["value"], got)
}
}
}
// nodeKeys turns the verifier's kid -> x map into the node's key set.
func nodeKeys(t *testing.T, doc any) StaticKeys {
t.Helper()
keys := StaticKeys{}
m, code := ctParseJWKS(doc)
if code != "" {
t.Fatalf("jwks: %s", code)
}
for kid, x := range m {
key, ok := publicKeyX(x)
if !ok {
t.Fatalf("key %s", kid)
}
keys[kid] = key
}
return keys
}
func verdictOf(err error) map[string]any {
if err != nil {
return map[string]any{"ok": false, "error": tokenWord(err)}
}
return map[string]any{"ok": true}
}
func atSeconds(s float64) time.Time { return time.Unix(0, int64(s*1e9)) }
func TestSessionVectorsGetTheSameVerdictFromTheNode(t *testing.T) {
for _, v := range ctList(t, ctVectors(t, "tokens.json"), "sessions") {
ctx := v["ctx"].(map[string]any)
token, isString := v["token"].(string)
var err error
if !isString {
err = tokFail(tokMalformed)
} else {
_, err = verifySession(token, atSeconds(ctx["now"].(float64)), nodeKeys(t, ctx["jwks"]), time.Duration(ctx["skew"].(float64))*time.Second)
}
ctCheckVerdict(t, v, verdictOf(err))
}
}
func TestProofVectorsGetTheSameVerdictFromTheNode(t *testing.T) {
for _, v := range ctList(t, ctVectors(t, "tokens.json"), "proofs") {
ctx := v["ctx"].(map[string]any)
now, skew := atSeconds(ctx["now"].(float64)), time.Duration(ctx["skew"].(float64))*time.Second
sessionToken := ctx["session"].(string)
session, err := verifySession(sessionToken, now, nodeKeys(t, ctx["jwks"]), skew)
if err != nil {
t.Fatalf("%v: session %v", v["id"], err)
}
pc, err := verifyProof(v["proof"].(string), now, sessionToken, session, ctx["nonce"].(string), ctx["htu"].(string), skew)
if err == nil {
// the replay check belongs to the caller: spend against the vector's set of used jtis
cache := newReplayCache(16)
for _, id := range ctStrings(ctx["seen"]) {
cache.spend(id, pc.IssuedAt, now, skew)
}
if !cache.spend(pc.JTI, pc.IssuedAt, now, skew) {
err = tokFail(tokReplayedJTI)
}
}
ctCheckVerdict(t, v, verdictOf(err))
}
}
func TestKeySetVectorsGetTheSameVerdictFromTheNode(t *testing.T) {
for _, v := range ctList(t, ctVectors(t, "tokens.json"), "key_sets") {
if v["kind"] != "jwks" {
continue
}
_, err := ParseJWKS([]byte(canonicalJSON(v["doc"])))
got := map[string]any{"ok": true}
if err != nil {
got = map[string]any{"ok": false, "error": err.Error()}
}
ctCheckVerdict(t, v, got)
}
}
func TestOKBodyBoundsTheWindowsToThePolicy(t *testing.T) {
big := okPolicy{PingIntervalS: 86400, DeadPeerS: 86400, StreamIdleS: 86400, StreamMaxAgeS: 86400, RenewBeforeValidUntilS: 86400}
got := boundWindows(big)
want := okPolicy{int64(ctPolInt("timing.ping_interval_s")), int64(ctPolInt("timing.dead_peer_s")), int64(ctPolInt("timing.stream_idle_s")),
int64(ctPolInt("timing.stream_max_age_s")), int64(ctPolInt("timing.renew_before_valid_until_s"))}
if got != want {
t.Fatalf("bounded %+v, want %+v", got, want)
}
small := okPolicy{PingIntervalS: 5, DeadPeerS: 9, StreamIdleS: 30, StreamMaxAgeS: 60, RenewBeforeValidUntilS: 10}
if boundWindows(small) != small {
t.Fatalf("a window below the policy's must stay as it is: %+v", boundWindows(small))
}
line := okBody{Node: "t1", Host: strings.Repeat("a", 20) + "." + testZone, Status: statusActive, ValidUntil: 1798766600, ServerTime: 1798761600, Policy: got}.line()
if m, err := parseLine(line, dirN2D); err != nil || formatLine(m) != line {
t.Fatalf("the OK line is not canonical: %q %v", line, err)
}
}
func TestAuthorizeAnswerIsCheckedBeforeItIsRelayed(t *testing.T) {
req := AuthorizeRequest{DeviceID: randomHex(16), SessionJTI: randomHex(12), Reason: "hello", Node: "t1"}
label := label20()
good := AuthorizeResponse{DeviceID: req.DeviceID, Label: label, Host: label + "." + testZone, Status: statusActive, Plan: "paid", ValidUntil: 1798766600,
ServerTime: 1798761600, Policy: okPolicy{20, 45, 1800, 14400, 300}}
if err := checkAuthorize(good, req); err != nil {
t.Fatal(err)
}
for name, mutate := range map[string]func(a *AuthorizeResponse){
"another device": func(a *AuthorizeResponse) { a.DeviceID = randomHex(16) },
"host without label": func(a *AuthorizeResponse) { a.Host = "evil." + testZone },
"host of another": func(a *AuthorizeResponse) { a.Host = label20() + "." + testZone },
"two labels deep": func(a *AuthorizeResponse) { a.Host = "x." + a.Host },
"unknown status": func(a *AuthorizeResponse) { a.Status = "free" },
"unknown plan": func(a *AuthorizeResponse) { a.Plan = "lifetime" },
"no valid_until": func(a *AuthorizeResponse) { a.ValidUntil = 0 },
"window of a week": func(a *AuthorizeResponse) { a.Policy.DeadPeerS = 7 * 86400 },
"zero window": func(a *AuthorizeResponse) { a.Policy.PingIntervalS = 0 },
"upper case host": func(a *AuthorizeResponse) { a.Host = strings.ToUpper(a.Host) },
"trailing dot host": func(a *AuthorizeResponse) { a.Host += "." },
"label not base32": func(a *AuthorizeResponse) { a.Label = strings.Repeat("0", 20); a.Host = a.Label + "." + testZone },
} {
a := good
mutate(&a)
if checkAuthorize(a, req) == nil {
t.Errorf("%s: accepted", name)
}
}
}
func TestReplayCacheRefusesAReusedJTIForTheProofsLifeAndIsBounded(t *testing.T) {
now := time.Unix(1798766600, 0)
skew := 5 * time.Second
c := newReplayCache(3)
if !c.spend("jti-aaaaaaaaaaaaaaaa", now.Unix(), now, skew) || c.spend("jti-aaaaaaaaaaaaaaaa", now.Unix(), now, skew) {
t.Fatal("a jti is single use")
}
// after the proof can no longer verify (its TTL plus the skew) the jti is forgotten and the cache makes room
later := now.Add(secs(ctPolInt("authorization.proof_ttl_s")) + skew + time.Second)
if !c.spend("jti-aaaaaaaaaaaaaaaa", later.Unix(), later, skew) {
t.Fatal("a forgotten jti is spendable again")
}
// full of live entries: a new proof is refused, never admitted unchecked
c = newReplayCache(2)
for _, id := range []string{"jti-1aaaaaaaaaaaaaaa", "jti-2aaaaaaaaaaaaaaa"} {
if !c.spend(id, now.Unix(), now, skew) {
t.Fatal(id)
}
}
if c.spend("jti-3aaaaaaaaaaaaaaa", now.Unix(), now, skew) {
t.Fatal("a full cache admitted a proof")
}
if !c.spend("jti-3aaaaaaaaaaaaaaa", later.Unix(), later, skew) {
t.Fatal("expired entries must free the room")
}
if c.size() != 1 {
t.Fatalf("size %d", c.size())
}
}
func FuzzParseLine(f *testing.F) {
for _, v := range []string{"HELLO 2 a.b.c", "PROOF a.b.c", "PING", "PONG", "HINT 192.0.2.4 ::1", "CLOSE", "CLOSE expired", "ERR auth", "CHALLENGE AAAAAAAAAAAAAAAAAAAAAA",
"ATTACH 00000000000000000000000000000000 AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "DRAIN 20", "OK {}", "OPEN x y z w"} {
f.Add(v, true)
f.Add(v, false)
}
f.Fuzz(func(t *testing.T, line string, toNode bool) {
dir := dirN2D
if toNode {
dir = dirD2N
}
m, err := parseLine(line, dir)
if err != nil {
if code := errCode(err); !inList(policyData.Protocol.ErrorCodes, code) {
t.Fatalf("%q: error word %q is not a protocol code", line, code)
}
return
}
// what the node accepts, it can write again unchanged in its canonical form, and that form parses to the same message
if canonical := formatLine(m); canonical != "" {
again, err := parseLine(canonical, dir)
if err != nil || !ctEqual(map[string]any(again), map[string]any(m)) {
t.Fatalf("%q: canonical %q reparses to %v (%v)", line, canonical, again, err)
}
}
})
}
func FuzzTokens(f *testing.F) {
doc, err := ctLoadFile(filepath.Join(ctVectorsDir(), "tokens.json"))
if err != nil {
f.Fatal(err)
}
for _, item := range doc.(map[string]any)["sessions"].([]any) {
if s, ok := item.(map[string]any)["token"].(string); ok {
f.Add(s)
}
}
for _, item := range doc.(map[string]any)["proofs"].([]any) {
f.Add(item.(map[string]any)["proof"].(string))
}
keys := StaticKeys{}
f.Fuzz(func(t *testing.T, token string) {
// no token, however broken, may panic the verifiers; a token no key signed never verifies
if _, err := verifySession(token, time.Unix(1798766600, 0), keys, 5*time.Second); err == nil {
t.Fatalf("a session verified against an empty key set: %q", token)
}
_, _ = splitJWS(token, 1<<16)
_, _ = verifyProof(token, time.Unix(1798766600, 0), "a.b.c", &SessionClaims{KeyX: strings.Repeat("A", 43)}, "n", "h", 5*time.Second)
})
}
files/tunnel/pump_buffers_test.go 98 lines · 2.9 KB
package main
import (
"net"
"sync/atomic"
"testing"
"time"
)
// An idle stream spends its life blocked in a read. The pump waits with a 4 KiB buffer and takes a 32 KiB one only for a read that filled the small one, giving it back when a
// read comes back short: an idle stream holds about 8 KiB of buffer, not 64 KiB (the chaos suite's memory-per-stream check).
func TestPump_AnIdleStreamHoldsNoBigBufferAndABulkTransferGetsOneAndGivesItBack(t *testing.T) {
e := newTestEnv(t, nil)
srcA, srcB := tcpPair(t)
dstA, dstB := tcpPair(t)
var ctr, total atomic.Int64
done := make(chan error, 1)
go func() { done <- e.srv.pump("s1", "up", dstA, srcB, &ctr, &total, nil, nil) }()
drain := make(chan int, 1)
go func() {
n := 0
buf := make([]byte, 1<<16)
for {
k, err := dstB.Read(buf)
n += k
if err != nil {
drain <- n
return
}
}
}()
time.Sleep(100 * time.Millisecond)
if got := e.srv.bigBuffersInUse.Load(); got != 0 {
t.Fatalf("an idle pump holds %d big buffers", got)
}
// a trickle never takes one
for i := 0; i < 5; i++ {
_, _ = srcA.Write([]byte("tiny"))
time.Sleep(20 * time.Millisecond)
}
if got := e.srv.bigBuffersInUse.Load(); got != 0 {
t.Fatalf("a trickle took %d big buffers", got)
}
// a bulk write fills the small buffer: the pump takes the big one for the following reads, and gives it back when a read comes back short
bulk := make([]byte, 1<<20)
var peak atomic.Int64
stopPoll := make(chan struct{})
go func() {
for {
select {
case <-stopPoll:
return
default:
if v := e.srv.bigBuffersInUse.Load(); v > peak.Load() {
peak.Store(v)
}
time.Sleep(100 * time.Microsecond)
}
}
}()
go func() { _, _ = srcA.Write(bulk) }()
eventually(t, "the bulk transfer went through", 5*time.Second, func() bool { return ctr.Load() >= int64(len(bulk)+20) })
close(stopPoll)
if peak.Load() < 1 {
t.Fatalf("a bulk transfer never took a big buffer: its throughput would be that of 4 KiB reads")
}
_, _ = srcA.Write([]byte("short"))
eventually(t, "the big buffer went back to the pool after a short read", 3*time.Second, func() bool { return e.srv.bigBuffersInUse.Load() == 0 })
_ = srcA.Close()
if err := <-done; err != nil {
t.Fatalf("a clean end of the source is not an error: %v", err)
}
_ = dstA.Close()
if n := <-drain; n != len(bulk)+20+5 {
t.Fatalf("the far side received %d bytes, want %d: the byte accounting changed", n, len(bulk)+25)
}
if got := e.srv.bigBuffersInUse.Load(); got != 0 {
t.Fatalf("%d big buffers leaked", got)
}
}
func tcpPair(t *testing.T) (a, b net.Conn) {
t.Helper()
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer ln.Close()
acc := make(chan net.Conn, 1)
go func() { c, _ := ln.Accept(); acc <- c }()
a, err = net.Dial("tcp", ln.Addr().String())
if err != nil {
t.Fatal(err)
}
b = <-acc
t.Cleanup(func() { _ = a.Close(); _ = b.Close() })
return a, b
}
files/tunnel/selfcheck.go 311 lines · 11.5 KB
package main
// Self-check (ALL 2.0, WP26; the design plan, section 6 point 4, the migration notes): before a computer is told to reconnect, somebody OUTSIDE the
// computer's own process dials its public name the way ChatGPT will, and says whether it works. The computer asks it of a node (the node is on the other side of
// the internet from the computer's own network), and the same function runs from the computer itself (the device's enrolment code) and from the operator's canary.
//
// dial the public address, SNI = the computer's host, TLS 1.2 offered and 1.3 required
// verify the chain against the public roots, the name and the validity at this moment
// compare the leaf's SubjectPublicKeyInfo hash with the one the computer expects (a certificate that is not the computer's own key is not the computer)
// GET /healthz through the same connection and read {"ok": true}
//
// The answer is contracts/schemas/selfcheck-result.schema.json: ok, the FIRST step that failed as one closed word (chain, expired, spki, name, tls_version, healthz,
// unreachable, timeout), the TLS version, the leaf's SPKI hash and its not_after. No host, no address, no free text: it can be shown, stored and sent as it is.
//
// What this file does NOT do: it does not decide who may ask, it has no listener of its own and it does not know the label registry. NewSelfCheckHandler is the
// HTTP shape (a POST with {v, label, expect}) for the operator's loopback port, with the per-label spacing and the cap on checks in flight that keep it from being
// a way to make the node dial itself; the route in server.go and the policy numbers (selfcheck.timeout_s, selfcheck.min_interval_s) are the node integration's (WP17).
import (
"bufio"
"context"
"crypto/sha256"
"crypto/tls"
"crypto/x509"
"encoding/base64"
"encoding/json"
"errors"
"io"
"net"
"net/http"
"os"
"strings"
"sync"
"time"
)
// The reasons, in the order the steps run. They are policy.enums.selfcheck_reason (the test pins that they are the same list).
const (
selfCheckChain = "chain"
selfCheckExpired = "expired"
selfCheckSPKI = "spki"
selfCheckName = "name"
selfCheckTLSVersion = "tls_version"
selfCheckHealthz = "healthz"
selfCheckUnreachable = "unreachable"
selfCheckTimeout = "timeout"
)
// SelfCheckResult is the contract's selfcheck-result.
type SelfCheckResult struct {
V int `json:"v"`
OK bool `json:"ok"`
Reason *string `json:"reason"`
TLSVersion *string `json:"tls_version"`
SPKISHA256 *string `json:"spki_sha256"`
NotAfter *int64 `json:"not_after"`
Healthz bool `json:"healthz"`
}
// SelfCheckConfig is what the check needs from outside. Nothing here has a default that is a number: the timeout is the caller's (the policy's).
type SelfCheckConfig struct {
// Dial opens the transport to the public address; nil is a plain TCP dial.
Dial func(ctx context.Context, network, addr string) (net.Conn, error)
// Roots are the public roots the chain must lead to; nil is the system's.
Roots *x509.CertPool
// Now is the moment validity is judged at; nil is the clock.
Now func() time.Time
// Timeout bounds the whole check (dial, handshake and healthz); zero leaves it to the context.
Timeout time.Duration
}
func selfCheckFail(res *SelfCheckResult, reason string) SelfCheckResult {
res.OK = false
res.Reason = &reason
return *res
}
// SelfCheck dials addr ("host:port", the public address of the node or of the computer's name), speaks TLS as `host`, and judges what it finds. `expect` is the set
// of SPKI hashes (SHA-256, base64url, no padding) the computer may be serving; empty means "any" (the operator's canary asks only whether the chain is good).
func SelfCheck(ctx context.Context, cfg SelfCheckConfig, addr, host string, expect []string) SelfCheckResult {
res := SelfCheckResult{V: 1}
if cfg.Timeout > 0 {
var cancel context.CancelFunc
ctx, cancel = context.WithTimeout(ctx, cfg.Timeout)
defer cancel()
}
dial := cfg.Dial
if dial == nil {
dial = (&net.Dialer{}).DialContext
}
conn, err := dial(ctx, "tcp", addr)
if err != nil {
return selfCheckFail(&res, networkReason(ctx, err))
}
defer conn.Close()
if deadline, ok := ctx.Deadline(); ok {
_ = conn.SetDeadline(deadline)
}
now := time.Now
if cfg.Now != nil {
now = cfg.Now
}
// Every step that can fail inside the handshake records its reason here before it refuses the connection: the standard library would otherwise report them all as one
// "certificate" error, and an operator needs to know which of the four it was.
var found string
verify := func(state tls.ConnectionState) error {
if len(state.PeerCertificates) == 0 {
found = selfCheckChain
return errors.New("no certificate")
}
leaf := state.PeerCertificates[0]
sum := sha256.Sum256(leaf.RawSubjectPublicKeyInfo)
spki := base64.RawURLEncoding.EncodeToString(sum[:])
notAfter := leaf.NotAfter.Unix()
res.SPKISHA256, res.NotAfter = &spki, ¬After
version := tlsVersionText(state.Version)
res.TLSVersion = &version
if state.Version != tls.VersionTLS13 {
found = selfCheckTLSVersion
return errors.New("tls version")
}
intermediates := x509.NewCertPool()
for _, extra := range state.PeerCertificates[1:] {
intermediates.AddCert(extra)
}
if _, verr := leaf.Verify(x509.VerifyOptions{DNSName: host, Roots: cfg.Roots, Intermediates: intermediates, CurrentTime: now()}); verr != nil {
found = verifyReason(verr)
return verr
}
if len(expect) > 0 && !containsText(expect, spki) {
found = selfCheckSPKI
return errors.New("spki")
}
return nil
}
secure := tls.Client(conn, &tls.Config{
ServerName: host,
MinVersion: tls.VersionTLS12,
NextProtos: []string{"http/1.1"},
// The verification is VerifyConnection above, so that every failure has its own reason; nothing is accepted that it refuses.
InsecureSkipVerify: true,
VerifyConnection: verify,
})
if err := secure.HandshakeContext(ctx); err != nil {
if found != "" {
return selfCheckFail(&res, found)
}
if strings.Contains(err.Error(), "protocol version") {
return selfCheckFail(&res, selfCheckTLSVersion)
}
return selfCheckFail(&res, networkReason(ctx, err))
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://"+host+"/healthz", nil)
if err != nil {
return selfCheckFail(&res, selfCheckHealthz)
}
req.Header.Set("User-Agent", "all-tunnel-selfcheck")
req.Close = true
if err := req.Write(secure); err != nil {
return selfCheckFail(&res, networkReason(ctx, err))
}
response, err := http.ReadResponse(bufio.NewReader(io.LimitReader(secure, 64<<10)), req)
if err != nil {
return selfCheckFail(&res, networkReasonOr(ctx, err, selfCheckHealthz))
}
defer response.Body.Close()
body, err := io.ReadAll(io.LimitReader(response.Body, 4<<10))
if err != nil {
return selfCheckFail(&res, networkReasonOr(ctx, err, selfCheckHealthz))
}
var health struct {
OK *bool `json:"ok"`
}
if response.StatusCode != http.StatusOK || json.Unmarshal(body, &health) != nil || health.OK == nil || !*health.OK {
return selfCheckFail(&res, selfCheckHealthz)
}
res.Healthz = true
res.OK = true
return res
}
func tlsVersionText(version uint16) string {
if version == tls.VersionTLS13 {
return "1.3"
}
return "1.2"
}
func containsText(list []string, text string) bool {
for _, item := range list {
if item == text {
return true
}
}
return false
}
// verifyReason is the first of the three things x509 checks that is wrong: the validity, the name, or the chain (an unknown authority, a bad signature, a certificate
// that is not for a server, anything else).
func verifyReason(err error) string {
var invalid x509.CertificateInvalidError
if errors.As(err, &invalid) && invalid.Reason == x509.Expired {
return selfCheckExpired
}
var wrongName x509.HostnameError
if errors.As(err, &wrongName) {
return selfCheckName
}
return selfCheckChain
}
func networkReason(ctx context.Context, err error) string {
return networkReasonOr(ctx, err, selfCheckUnreachable)
}
// networkReasonOr is `timeout` when the time ran out (the context, a deadline on the connection, a net.Error that says so) and `otherwise` for every other way a
// connection can end.
func networkReasonOr(ctx context.Context, err error, otherwise string) string {
var netErr net.Error
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, os.ErrDeadlineExceeded) || ctx.Err() != nil || (errors.As(err, &netErr) && netErr.Timeout()) {
return selfCheckTimeout
}
return otherwise
}
// ---- the HTTP shape --------------------------------------------------------------------------------------------------------------------------
// selfCheckRequest is {"v": 1, "label": "...", "expect": ["<spki>", ...]}; nothing else is accepted.
type selfCheckRequest struct {
V int `json:"v"`
Label string `json:"label"`
Expect []string `json:"expect"`
}
const (
selfCheckBodyMax = 1024
selfCheckExpectMax = 4
selfCheckSPKILength = 43
selfCheckInFlight = 4
)
// NewSelfCheckHandler is POST {v, label, expect} -> selfcheck-result, for a port that is not public (the operator's loopback listener). `lookup` says which host a label
// stands for on this node and whether its computer is connected (an unknown or offline label is an `unreachable` answer, never an error that tells which); `labelOK`
// is the label pattern of the policy. A label is checked at most once per `minInterval` and at most selfCheckInFlight checks run at once: a loop of requests is a
// 429, never a flood of dials.
func NewSelfCheckHandler(cfg SelfCheckConfig, addr string, lookup func(label string) (host string, ok bool), labelOK func(string) bool, minInterval time.Duration) http.Handler {
var mu sync.Mutex
last := map[string]time.Time{}
slots := make(chan struct{}, selfCheckInFlight)
reply := func(w http.ResponseWriter, status int, value any) {
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Cache-Control", "no-store")
w.Header().Set("X-Content-Type-Options", "nosniff")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(value)
}
refuse := func(w http.ResponseWriter, status int, word string) {
reply(w, status, map[string]any{"ok": false, "error": word})
}
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
refuse(w, http.StatusMethodNotAllowed, "invalid_request")
return
}
decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, selfCheckBodyMax))
decoder.DisallowUnknownFields()
var asked selfCheckRequest
if err := decoder.Decode(&asked); err != nil || decoder.More() || asked.V != 1 || !labelOK(asked.Label) || len(asked.Expect) > selfCheckExpectMax {
refuse(w, http.StatusBadRequest, "invalid_request")
return
}
for _, spki := range asked.Expect {
if len(spki) != selfCheckSPKILength {
refuse(w, http.StatusBadRequest, "invalid_request")
return
}
}
mu.Lock()
if at, seen := last[asked.Label]; seen && time.Since(at) < minInterval {
mu.Unlock()
refuse(w, http.StatusTooManyRequests, "busy")
return
}
select {
case slots <- struct{}{}:
last[asked.Label] = time.Now()
if len(last) > 4096 {
for label, at := range last {
if time.Since(at) >= minInterval {
delete(last, label)
}
}
}
mu.Unlock()
default:
mu.Unlock()
refuse(w, http.StatusTooManyRequests, "busy")
return
}
defer func() { <-slots }()
host, online := lookup(asked.Label)
if !online {
reason := selfCheckUnreachable
reply(w, http.StatusOK, SelfCheckResult{V: 1, Reason: &reason})
return
}
reply(w, http.StatusOK, SelfCheck(r.Context(), cfg, addr, host, asked.Expect))
})
}
files/tunnel/selfcheck_test.go 452 lines · 19.4 KB
package main
import (
"bytes"
"context"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/sha256"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"encoding/base64"
"encoding/json"
"fmt"
"math/big"
"net"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"time"
)
// ALL 2.0, WP26: the self-check. A real TLS server with a real certificate chain is dialed the way ChatGPT will, and the first step that fails is the answer: a wrong chain,
// an expired certificate, a key that is not the one the computer expects, a name that is not its own, a TLS version below 1.3, a /healthz that does not say ok, a port
// that is closed and a server that never answers. Every answer is the contract's selfcheck-result.
const selfCheckHost = "abcdefghijklmnopqrst.devices.all-e2e.test"
type selfCheckAuthority struct {
pool *x509.CertPool
cert *x509.Certificate
key *ecdsa.PrivateKey
}
func newSelfCheckAuthority(t *testing.T, name string) *selfCheckAuthority {
t.Helper()
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
t.Fatal(err)
}
template := &x509.Certificate{
SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: name}, NotBefore: time.Now().Add(-48 * time.Hour), NotAfter: time.Now().Add(240 * time.Hour),
IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign,
}
der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
if err != nil {
t.Fatal(err)
}
cert, err := x509.ParseCertificate(der)
if err != nil {
t.Fatal(err)
}
pool := x509.NewCertPool()
pool.AddCert(cert)
return &selfCheckAuthority{pool: pool, cert: cert, key: key}
}
type selfCheckLeaf struct {
certificate tls.Certificate
spki string
notAfter int64
}
// issue signs a leaf for `host` that is valid from `notBefore` to `notAfter`.
func (a *selfCheckAuthority) issue(t *testing.T, host string, notBefore, notAfter time.Time) selfCheckLeaf {
t.Helper()
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
t.Fatal(err)
}
template := &x509.Certificate{
SerialNumber: big.NewInt(time.Now().UnixNano()), Subject: pkix.Name{CommonName: host}, DNSNames: []string{host}, NotBefore: notBefore, NotAfter: notAfter,
KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
}
der, err := x509.CreateCertificate(rand.Reader, template, a.cert, &key.PublicKey, a.key)
if err != nil {
t.Fatal(err)
}
leaf, err := x509.ParseCertificate(der)
if err != nil {
t.Fatal(err)
}
sum := sha256.Sum256(leaf.RawSubjectPublicKeyInfo)
return selfCheckLeaf{certificate: tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key}, spki: base64.RawURLEncoding.EncodeToString(sum[:]), notAfter: leaf.NotAfter.Unix()}
}
func (a *selfCheckAuthority) good(t *testing.T) selfCheckLeaf {
return a.issue(t, selfCheckHost, time.Now().Add(-time.Hour), time.Now().Add(45*24*time.Hour))
}
// serveSelfCheck runs a TLS server (HTTP/1.1 behind it) on a loopback port with the certificate and the TLS limits given, and returns its address.
func serveSelfCheck(t *testing.T, leaf selfCheckLeaf, maxVersion uint16, healthz http.HandlerFunc) string {
t.Helper()
mux := http.NewServeMux()
mux.HandleFunc("/healthz", healthz)
server := httptest.NewUnstartedServer(mux)
server.TLS = &tls.Config{Certificates: []tls.Certificate{leaf.certificate}, MinVersion: tls.VersionTLS12, MaxVersion: maxVersion}
server.StartTLS()
t.Cleanup(server.Close)
return server.Listener.Addr().String()
}
func okHealthz(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"ok":true}`))
}
func checkWith(t *testing.T, authority *selfCheckAuthority, addr, host string, expect []string, change func(*SelfCheckConfig)) SelfCheckResult {
t.Helper()
cfg := SelfCheckConfig{Roots: authority.pool, Timeout: 5 * time.Second}
if change != nil {
change(&cfg)
}
result := SelfCheck(context.Background(), cfg, addr, host, expect)
// every answer, whatever it says, is the contract and carries no host and no address
encoded, err := json.Marshal(result)
if err != nil {
t.Fatal(err)
}
var instance any
if err := json.Unmarshal(encoded, &instance); err != nil {
t.Fatal(err)
}
if problems := ctLoadSchemas().validate("selfcheck-result.schema.json", instance); len(problems) != 0 {
t.Fatalf("not the contract: %v in %s", problems, encoded)
}
if strings.Contains(string(encoded), host) || strings.Contains(string(encoded), "127.0.0.1") {
t.Fatalf("the answer names a host or an address: %s", encoded)
}
return result
}
func reasonOf(r SelfCheckResult) string {
if r.Reason == nil {
return ""
}
return *r.Reason
}
func TestSelfCheckPassesForAGoodChainTheRightKeyAndAHealthyPath(t *testing.T) {
authority := newSelfCheckAuthority(t, "good root")
leaf := authority.good(t)
addr := serveSelfCheck(t, leaf, tls.VersionTLS13, okHealthz)
result := checkWith(t, authority, addr, selfCheckHost, []string{leaf.spki}, nil)
if !result.OK || result.Reason != nil || !result.Healthz {
t.Fatalf("a good path: %+v", result)
}
if result.TLSVersion == nil || *result.TLSVersion != "1.3" || result.SPKISHA256 == nil || *result.SPKISHA256 != leaf.spki || result.NotAfter == nil || *result.NotAfter != leaf.notAfter {
t.Fatalf("what it saw: %+v", result)
}
// without an expectation (the operator's canary asks only whether the chain is good) the key is not compared
if anyKey := checkWith(t, authority, addr, selfCheckHost, nil, nil); !anyKey.OK {
t.Fatalf("no expectation: %+v", anyKey)
}
// the expected set may hold the key of the generation that is about to replace this one
if two := checkWith(t, authority, addr, selfCheckHost, []string{strings.Repeat("A", 43), leaf.spki}, nil); !two.OK {
t.Fatalf("a set of two: %+v", two)
}
}
func TestSelfCheckFailsForAChainThatDoesNotLeadToAPublicRoot(t *testing.T) {
trusted := newSelfCheckAuthority(t, "the root the checker trusts")
stranger := newSelfCheckAuthority(t, "another root")
addr := serveSelfCheck(t, stranger.good(t), tls.VersionTLS13, okHealthz)
result := checkWith(t, trusted, addr, selfCheckHost, nil, nil)
if result.OK || reasonOf(result) != selfCheckChain || result.Healthz {
t.Fatalf("a wrong chain: %+v", result)
}
// and a self-signed leaf with no authority at all
selfSigned := trusted.issue(t, selfCheckHost, time.Now().Add(-time.Hour), time.Now().Add(time.Hour))
selfSigned.certificate.Certificate = selfSigned.certificate.Certificate[:1]
key, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
template := &x509.Certificate{SerialNumber: big.NewInt(7), Subject: pkix.Name{CommonName: selfCheckHost}, DNSNames: []string{selfCheckHost}, NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour)}
der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
if err != nil {
t.Fatal(err)
}
own := serveSelfCheck(t, selfCheckLeaf{certificate: tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key}}, tls.VersionTLS13, okHealthz)
if again := checkWith(t, trusted, own, selfCheckHost, nil, nil); reasonOf(again) != selfCheckChain {
t.Fatalf("a self-signed leaf: %+v", again)
}
}
func TestSelfCheckFailsForAnExpiredCertificateAndForOneThatIsNotYetValid(t *testing.T) {
authority := newSelfCheckAuthority(t, "root")
expired := authority.issue(t, selfCheckHost, time.Now().Add(-90*24*time.Hour), time.Now().Add(-time.Hour))
result := checkWith(t, authority, serveSelfCheck(t, expired, tls.VersionTLS13, okHealthz), selfCheckHost, []string{expired.spki}, nil)
if result.OK || reasonOf(result) != selfCheckExpired {
t.Fatalf("expired: %+v", result)
}
if result.NotAfter == nil || *result.NotAfter != expired.notAfter {
t.Fatalf("the answer still says when it expired: %+v", result)
}
future := authority.issue(t, selfCheckHost, time.Now().Add(24*time.Hour), time.Now().Add(48*time.Hour))
if early := checkWith(t, authority, serveSelfCheck(t, future, tls.VersionTLS13, okHealthz), selfCheckHost, nil, nil); reasonOf(early) != selfCheckExpired {
t.Fatalf("not yet valid: %+v", early)
}
// validity is judged at the moment the check says: a good certificate seen ten years from now is an expired one
good := authority.good(t)
later := checkWith(t, authority, serveSelfCheck(t, good, tls.VersionTLS13, okHealthz), selfCheckHost, nil, func(cfg *SelfCheckConfig) {
cfg.Now = func() time.Time { return time.Now().Add(10 * 365 * 24 * time.Hour) }
})
if reasonOf(later) != selfCheckExpired {
t.Fatalf("in ten years: %+v", later)
}
}
func TestSelfCheckFailsForAKeyThatIsNotTheOneTheComputerExpects(t *testing.T) {
authority := newSelfCheckAuthority(t, "root")
leaf := authority.good(t)
other := authority.good(t)
addr := serveSelfCheck(t, leaf, tls.VersionTLS13, okHealthz)
result := checkWith(t, authority, addr, selfCheckHost, []string{other.spki}, nil)
if result.OK || reasonOf(result) != selfCheckSPKI || result.Healthz {
t.Fatalf("a mismatching key: %+v", result)
}
if result.SPKISHA256 == nil || *result.SPKISHA256 != leaf.spki {
t.Fatalf("the answer says which key it saw (so the operator can compare): %+v", result)
}
}
func TestSelfCheckFailsForANameThatIsNotTheComputersOwn(t *testing.T) {
authority := newSelfCheckAuthority(t, "root")
wrong := authority.issue(t, "tsrqponmlkjihgfedcba.devices.all-e2e.test", time.Now().Add(-time.Hour), time.Now().Add(time.Hour))
result := checkWith(t, authority, serveSelfCheck(t, wrong, tls.VersionTLS13, okHealthz), selfCheckHost, []string{wrong.spki}, nil)
if result.OK || reasonOf(result) != selfCheckName {
t.Fatalf("a certificate for another label: %+v", result)
}
}
func TestSelfCheckRequiresTLS13(t *testing.T) {
authority := newSelfCheckAuthority(t, "root")
leaf := authority.good(t)
result := checkWith(t, authority, serveSelfCheck(t, leaf, tls.VersionTLS12, okHealthz), selfCheckHost, []string{leaf.spki}, nil)
if result.OK || reasonOf(result) != selfCheckTLSVersion || result.TLSVersion == nil || *result.TLSVersion != "1.2" {
t.Fatalf("a TLS 1.2 server: %+v", result)
}
}
func TestSelfCheckFailsWhenHealthzDoesNotSayOk(t *testing.T) {
authority := newSelfCheckAuthority(t, "root")
leaf := authority.good(t)
for name, handler := range map[string]http.HandlerFunc{
"a server error": func(w http.ResponseWriter, _ *http.Request) { http.Error(w, "down", http.StatusInternalServerError) },
"not ok": func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte(`{"ok":false}`)) },
"not json": func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte(`<html>welcome</html>`)) },
"no member": func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte(`{}`)) },
"a redirect": func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "/elsewhere", http.StatusFound) },
} {
result := checkWith(t, authority, serveSelfCheck(t, leaf, tls.VersionTLS13, handler), selfCheckHost, []string{leaf.spki}, nil)
if result.OK || reasonOf(result) != selfCheckHealthz || result.Healthz {
t.Fatalf("%s: %+v", name, result)
}
if result.SPKISHA256 == nil {
t.Fatalf("%s: the TLS part passed, so the answer still carries what it saw", name)
}
}
}
func TestSelfCheckSaysUnreachableForAClosedPortAndTimeoutForAServerThatNeverAnswers(t *testing.T) {
authority := newSelfCheckAuthority(t, "root")
closed, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
address := closed.Addr().String()
closed.Close()
if result := checkWith(t, authority, address, selfCheckHost, nil, nil); result.OK || reasonOf(result) != selfCheckUnreachable {
t.Fatalf("a closed port: %+v", result)
}
silent, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer silent.Close()
var mu sync.Mutex
var held []net.Conn
go func() {
for {
conn, err := silent.Accept()
if err != nil {
return
}
mu.Lock()
held = append(held, conn)
mu.Unlock()
}
}()
defer func() {
mu.Lock()
for _, conn := range held {
conn.Close()
}
mu.Unlock()
}()
started := time.Now()
result := checkWith(t, authority, silent.Addr().String(), selfCheckHost, nil, func(cfg *SelfCheckConfig) { cfg.Timeout = 300 * time.Millisecond })
if result.OK || reasonOf(result) != selfCheckTimeout {
t.Fatalf("a server that never answers: %+v", result)
}
if elapsed := time.Since(started); elapsed > 3*time.Second {
t.Fatalf("the timeout did not bound the check: %v", elapsed)
}
// a server that answers the handshake and then says nothing about /healthz is a timeout too, not a hang
leaf := authority.good(t)
stalled := serveSelfCheck(t, leaf, tls.VersionTLS13, func(w http.ResponseWriter, r *http.Request) { <-r.Context().Done() })
started = time.Now()
hung := checkWith(t, authority, stalled, selfCheckHost, []string{leaf.spki}, func(cfg *SelfCheckConfig) { cfg.Timeout = 400 * time.Millisecond })
if hung.OK || reasonOf(hung) != selfCheckTimeout || time.Since(started) > 3*time.Second {
t.Fatalf("a healthz that never answers: %+v after %v", hung, time.Since(started))
}
}
func TestSelfCheckReasonsAreThePolicysEnumInTheOrderTheStepsRun(t *testing.T) {
policy := ctPolStrings("enums.selfcheck_reason")
ours := []string{selfCheckChain, selfCheckExpired, selfCheckSPKI, selfCheckName, selfCheckTLSVersion, selfCheckHealthz, selfCheckUnreachable, selfCheckTimeout}
if fmt.Sprint(policy) != fmt.Sprint(ours) {
t.Fatalf("policy %v, code %v", policy, ours)
}
}
func TestSelfCheckTunnelsThroughTheDialerItIsGiven(t *testing.T) {
// the node dials its own public path through whatever Dial gives: here, a function that ignores the address and goes to the test server
authority := newSelfCheckAuthority(t, "root")
leaf := authority.good(t)
real := serveSelfCheck(t, leaf, tls.VersionTLS13, okHealthz)
var dialed string
result := checkWith(t, authority, "203.0.113.9:443", selfCheckHost, []string{leaf.spki}, func(cfg *SelfCheckConfig) {
cfg.Dial = func(ctx context.Context, network, addr string) (net.Conn, error) {
dialed = addr
return (&net.Dialer{}).DialContext(ctx, network, real)
}
})
if !result.OK || dialed != "203.0.113.9:443" {
t.Fatalf("%+v dialed %q", result, dialed)
}
}
// ---- the HTTP shape ----------------------------------------------------------------------------------------------------------------------------
func selfCheckPost(handler http.Handler, body string) *httptest.ResponseRecorder {
request := httptest.NewRequest(http.MethodPost, "/selfcheck", bytes.NewBufferString(body))
recorder := httptest.NewRecorder()
handler.ServeHTTP(recorder, request)
return recorder
}
func TestSelfCheckHandlerAnswersTheContractAndRefusesWhatItDoesNotKnow(t *testing.T) {
authority := newSelfCheckAuthority(t, "root")
leaf := authority.good(t)
addr := serveSelfCheck(t, leaf, tls.VersionTLS13, okHealthz)
cfg := SelfCheckConfig{Roots: authority.pool, Timeout: 5 * time.Second}
labelOK := func(label string) bool { return ctLabelOK(label) }
lookup := func(label string) (string, bool) {
return label + ".devices.all-e2e.test", label == "abcdefghijklmnopqrst"
}
handler := NewSelfCheckHandler(cfg, addr, lookup, labelOK, 0)
good := selfCheckPost(handler, fmt.Sprintf(`{"v":1,"label":"abcdefghijklmnopqrst","expect":["%s"]}`, leaf.spki))
var answer map[string]any
if good.Code != http.StatusOK || json.Unmarshal(good.Body.Bytes(), &answer) != nil || answer["ok"] != true {
t.Fatalf("%d %s", good.Code, good.Body.String())
}
if problems := ctLoadSchemas().validate("selfcheck-result.schema.json", answer); len(problems) != 0 {
t.Fatalf("not the contract: %v", problems)
}
if good.Header().Get("Cache-Control") != "no-store" {
t.Fatal("an answer about one computer is never cached")
}
// an unknown or offline label is the same kind of answer as any other failure: unreachable, never a different status that tells which labels exist
offline := selfCheckPost(handler, `{"v":1,"label":"tsrqponmlkjihgfedcba","expect":[]}`)
var gone map[string]any
if offline.Code != http.StatusOK || json.Unmarshal(offline.Body.Bytes(), &gone) != nil || gone["ok"] != false || gone["reason"] != "unreachable" {
t.Fatalf("%d %s", offline.Code, offline.Body.String())
}
for _, body := range []string{
``, `{`, `[]`, `{"v":2,"label":"abcdefghijklmnopqrst","expect":[]}`, `{"v":1,"label":"short","expect":[]}`, `{"v":1,"label":"ABCDEFGHIJKLMNOPQRST","expect":[]}`,
`{"v":1,"label":"abcdefghijklmnopqrst","expect":["short"]}`, `{"v":1,"label":"abcdefghijklmnopqrst","expect":[],"host":"evil.example"}`,
`{"v":1,"label":"abcdefghijklmnopqrst","expect":[]} {}`, fmt.Sprintf(`{"v":1,"label":"abcdefghijklmnopqrst","expect":["%[1]s","%[1]s","%[1]s","%[1]s","%[1]s"]}`, strings.Repeat("A", 43)),
`{"v":1,"label":"abcdefghijklmnopqrst","expect":[],"pad":"` + strings.Repeat("x", 2000) + `"}`,
} {
if refused := selfCheckPost(handler, body); refused.Code != http.StatusBadRequest {
t.Fatalf("%q: %d %s", truncate(body), refused.Code, refused.Body.String())
}
}
for _, method := range []string{http.MethodGet, http.MethodPut, http.MethodDelete} {
recorder := httptest.NewRecorder()
handler.ServeHTTP(recorder, httptest.NewRequest(method, "/selfcheck", nil))
if recorder.Code != http.StatusMethodNotAllowed {
t.Fatalf("%s: %d", method, recorder.Code)
}
}
}
func TestSelfCheckHandlerSpacesTheChecksOfOneLabelAndCapsTheOnesInFlight(t *testing.T) {
authority := newSelfCheckAuthority(t, "root")
leaf := authority.good(t)
addr := serveSelfCheck(t, leaf, tls.VersionTLS13, okHealthz)
cfg := SelfCheckConfig{Roots: authority.pool, Timeout: 5 * time.Second}
lookup := func(label string) (string, bool) { return selfCheckHost, true }
spaced := NewSelfCheckHandler(cfg, addr, lookup, ctLabelOK, time.Hour)
body := func(label string) string {
return fmt.Sprintf(`{"v":1,"label":"%s","expect":["%s"]}`, label, leaf.spki)
}
if first := selfCheckPost(spaced, body("abcdefghijklmnopqrst")); first.Code != http.StatusOK {
t.Fatalf("%d %s", first.Code, first.Body.String())
}
if again := selfCheckPost(spaced, body("abcdefghijklmnopqrst")); again.Code != http.StatusTooManyRequests || !strings.Contains(again.Body.String(), `"busy"`) {
t.Fatalf("a second check of the same label within the spacing: %d %s", again.Code, again.Body.String())
}
if other := selfCheckPost(spaced, body("tsrqponmlkjihgfedcba")); other.Code != http.StatusOK {
t.Fatalf("another label is its own: %d", other.Code)
}
// at most selfCheckInFlight checks run at once, whatever labels they are for: the rest are busy, none waits
release := make(chan struct{})
started := make(chan struct{}, 16)
slow := func(ctx context.Context, network, addr string) (net.Conn, error) {
started <- struct{}{}
select {
case <-release:
case <-ctx.Done():
}
return nil, context.Canceled
}
capped := NewSelfCheckHandler(SelfCheckConfig{Dial: slow, Roots: authority.pool, Timeout: 5 * time.Second}, addr, lookup, ctLabelOK, 0)
var wg sync.WaitGroup
results := make(chan int, 16)
labels := []string{"aaaaaaaaaaaaaaaaaaaa", "bbbbbbbbbbbbbbbbbbbb", "cccccccccccccccccccc", "dddddddddddddddddddd", "eeeeeeeeeeeeeeeeeeee", "ffffffffffffffffffff"}
for _, label := range labels[:selfCheckInFlight] {
wg.Add(1)
go func(label string) {
defer wg.Done()
results <- selfCheckPost(capped, body(label)).Code
}(label)
}
for range labels[:selfCheckInFlight] {
<-started
}
if extra := selfCheckPost(capped, body(labels[selfCheckInFlight])); extra.Code != http.StatusTooManyRequests {
t.Fatalf("one more than the cap: %d", extra.Code)
}
close(release)
wg.Wait()
close(results)
for code := range results {
if code != http.StatusOK {
t.Fatalf("the checks in flight finish normally: %d", code)
}
}
}
files/tunnel/server.go 1894 lines · 66.0 KB
package main
import (
"bufio"
"context"
"crypto/rand"
"crypto/subtle"
"crypto/tls"
"encoding/hex"
"errors"
"fmt"
"hash/maphash"
"io"
"log/slog"
"net"
"net/netip"
"os"
"strings"
"sync"
"sync/atomic"
"time"
)
const (
// TLS alert descriptions, sent (in the clear, as a fatal alert record) to a client we decide not to serve.
alertInternalError = 80
alertUnrecognizedName = 112
)
var (
errOffline = errors.New("device_offline")
errAttachTimeout = errors.New("attach_timeout")
errControlWrite = errors.New("control_write_failed")
errShuttingDown = errors.New("shutting_down")
errByteCap = errors.New("stream byte cap reached")
errHostTaken = errors.New("host_taken")
errGatedBytes = errors.New("gated byte budget spent")
)
// pendingStream is an OPEN waiting for its ATTACH: the secret is one use, and only the device that was sent the OPEN knows it.
type pendingStream struct {
cc *ctrl
secret string
ch chan net.Conn // buffered(1): the ATTACHed data connection
}
// Server is the tunnel node. It never terminates a client's TLS: the only TLS it speaks is the one it holds for its own name, with devices.
type Server struct {
cfg Config
log *slog.Logger
tlsCfg *tls.Config
routes *routeTable
replay *replayCache
nodeURL string // the htu of a proof: this node's URL
clk clock
health *cpHealth
ctx context.Context
cancel context.CancelFunc
bg sync.WaitGroup // the feed loop, the background tasks and the refreshes of connections: ended by cancel
mu sync.Mutex
pending map[string]*pendingStream
conns map[*ctrl]struct{} // every live control connection, for shutdown
streams atomic.Int64 // client connections being served, global
anonStreams atomic.Int64 // of them, the ones whose source is neither openai nor marked good: they may not use the share reserved for the others
unauth atomic.Int64 // connections that have not finished their first message
anonUnauth atomic.Int64 // of them, the ones whose source is neither ChatGPT's, nor a device's, nor marked good: they may not use the share that is kept for the others
authQ atomic.Int64 // exchanges that wait for, or are in, a call to authorize
connSeq atomic.Int64
authSem chan struct{} // the calls to authorize in flight: at most cfg.AuthorizeMax
watchMu sync.Mutex
watches map[*feedWatch]struct{} // the exchanges waiting for authorize, each told of the feed events applied meanwhile
killed atomic.Bool // the global kill switch is on: no device is served until it is cleared
draining atomic.Bool // SIGTERM: no new client and no new device connection, the open streams finish
drainFor atomic.Int64 // the quiet period this drain was started with, in nanoseconds (what a late device is told)
// hardening (limits.go, hints.go, gated.go, openai_prefixes.go, metrics.go, observer.go)
ips *ipLimits
hints *hintTable
openai *openaiPrefixes
gates *gateTable
met *metrics
sampler *logSampler
seed maphash.Seed // per boot: what Server.short hashes with, so two log lines of one source can be told apart and nothing can be read back
startedAt time.Time
httpsLn, httpLn net.Listener
metricsSrv *metricsServer
wg sync.WaitGroup
bufPool sync.Pool // the 32 KiB buffers a pump takes for bulk reads
smallBufPool sync.Pool // the 4 KiB buffers a pump waits in a read with
bigBuffersInUse atomic.Int64 // 32 KiB buffers held by a pump now (an idle stream holds none: tests and the memory check read it)
}
const (
pumpSmallBuf = 4 << 10
pumpBigBuf = 32 << 10
)
// NewServer validates cfg and builds the node. It does not listen yet.
func NewServer(cfg Config) (*Server, error) {
if cfg.Logger == nil {
cfg.Logger = slog.New(slog.NewJSONHandler(io.Discard, nil))
}
if cfg.Now == nil {
cfg.Now = time.Now
}
if cfg.Health == nil {
cfg.Health = &cpHealth{}
}
if err := cfg.validate(); err != nil {
return nil, err
}
ctx, cancel := context.WithCancel(context.Background())
clk := newClock(cfg.Now, cfg.AfterFunc)
s := &Server{
cfg: cfg,
log: cfg.Logger,
routes: newRouteTable(),
replay: newReplayCache(replayLimit),
nodeURL: nodeProofURL(cfg.NodeHost, cfg.PublicPort),
clk: clk,
health: cfg.Health,
ctx: ctx,
cancel: cancel,
pending: map[string]*pendingStream{},
conns: map[*ctrl]struct{}{},
watches: map[*feedWatch]struct{}{},
authSem: make(chan struct{}, cfg.AuthorizeMax),
seed: maphash.MakeSeed(),
met: newMetrics(),
sampler: newLogSampler(clk),
hints: newHintTable(clk, cfg.HintTTL, cfg.HintsPerLabel),
gates: newGateTable(clk),
startedAt: clk.Now(),
}
s.ips = newIPLimits(&cfg, clk, s.seed)
s.openai = newOpenAIPrefixes(&cfg, clk, s.met.prefixRefresh.inc)
s.routes.lim = labelLimitsOf(&cfg)
if cfg.HTTPSAddr != "" {
s.tlsCfg = &tls.Config{
GetCertificate: cfg.CertSource,
MinVersion: tls.VersionTLS13,
NextProtos: []string{policyData.Protocol.ALPN},
}
}
s.bufPool.New = func() any { b := make([]byte, pumpBigBuf); return &b }
s.smallBufPool.New = func() any { b := make([]byte, pumpSmallBuf); return &b }
return s, nil
}
// replayLimit bounds the proofs remembered at once (a proof lives for the policy's proof TTL, so this is the number of proofs per minute the node
// admits before it refuses more).
const replayLimit = 1 << 18
func (s *Server) now() time.Time { return s.clk.Now() }
func addrOf(ln net.Listener) string {
if ln == nil {
return ""
}
return ln.Addr().String()
}
func (s *Server) HTTPSAddr() string { return addrOf(s.httpsLn) }
func (s *Server) HTTPAddr() string { return addrOf(s.httpLn) }
// Online reports whether the host has a live control connection that opens streams.
func (s *Server) Online(host string) bool {
cc, _ := s.routes.lookup(host)
return cc != nil && cc.canOpen(s.now())
}
// Start opens the two listeners (443 and 80: there is no other) and serves until Shutdown.
func (s *Server) Start() error {
// Every socket the node accepts gets TCP keepalive from the policy: a stream whose peer vanished (a computer that slept, a network that changed) ends
// by itself instead of holding its slot until the idle timeout.
lc := s.listenConfig()
listen := func(addr string) (net.Listener, error) {
if addr == "" {
return nil, nil
}
return lc.Listen(context.Background(), "tcp", addr)
}
var err error
if s.httpsLn, err = listen(s.cfg.HTTPSAddr); err != nil {
return fmt.Errorf("https listener: %w", err)
}
if s.httpLn, err = listen(s.cfg.HTTPAddr); err != nil {
_ = s.closeListeners()
return fmt.Errorf("http listener: %w", err)
}
if merr := s.startMetrics(); merr != nil {
// The node serves without its metrics rather than not at all: a port that is taken is not a reason for a fleet's devices to lose their node.
// The operator sees the line and the missing /healthz.
s.log.Warn("metrics_unavailable")
}
// What the node must already have stopped serving is known before the first device connects: the feed's first answer (a revoke that happened while
// the node was down, the kill switch) comes first, bounded so that a late control plane does not keep the node from starting.
s.startBackground()
for _, l := range []struct {
ln net.Listener
fn func(net.Conn)
}{{s.httpsLn, s.handleHTTPS}, {s.httpLn, s.handleHTTP}} {
if l.ln != nil {
s.wg.Add(1)
go s.acceptLoop(l.ln, l.fn)
}
}
s.log.Info("listening", "https", s.HTTPSAddr(), "http", s.HTTPAddr(), "node", s.cfg.NodeID)
return nil
}
// keepaliveProbes is how many unanswered keepalive probes end a connection.
const keepaliveProbes = 4
// listenConfig is how the node's listeners are made: the connections they accept have TCP keepalive on, with the policy's period.
func (s *Server) listenConfig() net.ListenConfig {
return net.ListenConfig{KeepAliveConfig: net.KeepAliveConfig{Enable: true, Idle: s.cfg.TCPKeepalive, Interval: s.cfg.TCPKeepalive, Count: keepaliveProbes}}
}
func (s *Server) closeListeners() error {
var first error
for _, ln := range []net.Listener{s.httpsLn, s.httpLn} {
if ln != nil {
if err := ln.Close(); err != nil && first == nil {
first = err
}
}
}
return first
}
// Shutdown stops accepting, tells every device the node is going (CLOSE shutdown) and waits (up to grace) for streams to finish; the streams are
// force-closed when it ends.
func (s *Server) Shutdown(grace time.Duration) { s.shutdown(grace, closeShutdown) }
func (s *Server) shutdown(grace time.Duration, reason string) {
_ = s.closeListeners()
s.stopMetrics()
s.mu.Lock()
live := make([]*ctrl, 0, len(s.conns))
for cc := range s.conns {
live = append(live, cc)
}
s.mu.Unlock()
for _, cc := range live {
cc.close(reason)
}
done := make(chan struct{})
go func() { s.wg.Wait(); close(done) }()
select {
case <-done:
case <-time.After(grace):
}
s.mu.Lock() // with goBackground: a task is either counted before the wait below starts, or sees the node ended and does not start
s.cancel()
s.mu.Unlock()
select {
case <-done:
case <-time.After(2 * time.Second):
}
bgDone := make(chan struct{})
go func() { s.bg.Wait(); close(bgDone) }()
select {
case <-bgDone:
case <-time.After(2 * time.Second):
}
}
func (s *Server) acceptLoop(ln net.Listener, handle func(net.Conn)) {
defer s.wg.Done()
var delay time.Duration
for {
c, err := ln.Accept()
if err != nil {
if s.ctx.Err() != nil || errors.Is(err, net.ErrClosed) {
return
}
delay = min(max(2*delay, 5*time.Millisecond), time.Second)
s.log.Warn("accept_error", "retry_in_ms", delay.Milliseconds())
select {
case <-time.After(delay):
case <-s.ctx.Done():
return
}
continue
}
delay = 0
s.wg.Add(1)
go handle(c)
}
}
func tryAcquire(counter *atomic.Int64, limit int64) bool {
for {
cur := counter.Load()
if cur >= limit {
return false
}
if counter.CompareAndSwap(cur, cur+1) {
return true
}
}
}
// anonUnauthMax is the most unauthenticated connections the sources nobody vouches for may hold together.
func (s *Server) anonUnauthMax() int64 {
return int64(s.cfg.MaxUnauth - s.cfg.MaxUnauth*s.cfg.OpenAIReservedPct/100)
}
func newStreamID() string {
var b [16]byte
if _, err := rand.Read(b[:]); err != nil {
panic("crypto/rand failed: " + err.Error())
}
return hex.EncodeToString(b[:])
}
// newSecret is a 256-bit one-use attach secret (43 base64url characters).
func newSecret() string {
var b [32]byte
if _, err := rand.Read(b[:]); err != nil {
panic("crypto/rand failed: " + err.Error())
}
return b64Encode(b[:])
}
// newNonce is the 128-bit CHALLENGE nonce (22 base64url characters).
func newNonce() string {
var b [16]byte
if _, err := rand.Read(b[:]); err != nil {
panic("crypto/rand failed: " + err.Error())
}
return b64Encode(b[:])
}
func (s *Server) tapBytes(id, dir string, p []byte) {
if s.cfg.tap != nil {
s.cfg.tap(id, dir, p)
}
}
// emit records one protocol step for the contracts' transcript checker; it costs nothing when no trace is set.
func (s *Server) emit(at time.Time, ev map[string]any) {
if s.cfg.trace != nil {
ev["t"] = epochSeconds(at)
s.cfg.trace(ev)
}
}
// sendAlert writes one fatal TLS alert record, then lets the peer drain for a moment so it is not reset before it can read the alert.
func sendAlert(c net.Conn, desc byte) {
_ = c.SetWriteDeadline(time.Now().Add(time.Second))
_, _ = c.Write([]byte{21, 3, 3, 0, 2, 2, desc})
if cw, ok := c.(interface{ CloseWrite() error }); ok {
_ = cw.CloseWrite()
}
_ = c.SetReadDeadline(time.Now().Add(time.Second))
_, _ = io.Copy(io.Discard, io.LimitReader(c, 16<<10))
}
func helloReason(err error) string {
switch {
case errors.Is(err, errNotTLS):
return "not_tls"
case errors.Is(err, errNotClientHello), errors.Is(err, errHelloMalformed):
return "bad_hello"
case errors.Is(err, errHelloTooLarge):
return "hello_too_large"
case errors.Is(err, os.ErrDeadlineExceeded):
return "hello_timeout"
case errors.Is(err, io.EOF), errors.Is(err, io.ErrUnexpectedEOF):
return "closed_before_hello"
default:
return "read_error"
}
}
// peerAddr is the client's address as the contracts write it in an OPEN: no zone, an IPv4-mapped address as the IPv4 address it carries.
func peerAddr(c net.Conn) (netip.Addr, bool) {
ap, err := netip.ParseAddrPort(c.RemoteAddr().String())
if err != nil {
return netip.Addr{}, false
}
return ap.Addr().WithZone("").Unmap(), true
}
// ---------------------------------------------------------------- routing table
// hostEntry is what the node knows about one device host: the control connection that serves it (nil while the device is away) and how many client
// streams are open on it.
type hostEntry struct {
cc *ctrl
streams int64 // client streams open on the host, all slices together
anon int64 // of them, the ones in the anonymous slice (the rest are in the slice of sources the device marked good)
live map[*liveStream]struct{} // the client streams being served on this host: what a revoke, a pause or an expiry must be able to cut
gated atomic.Bool // the control plane's last status for the host's connection is gated: its streams are held to the day's budget (gated.go)
newOnes bucket // the host's budget of new streams per second, for sources that are not marked good
}
// liveStream is one client stream the node serves for a host, from the moment it is admitted (before its OPEN) to its end. A revoke, a pause, an
// expired connection or the kill switch kills it: whichever connections it holds are closed, and one that is not attached yet is closed the moment it is.
type liveStream struct {
mu sync.Mutex
client net.Conn
dev net.Conn
killed string // why it was killed, "" while it lives
he *hostEntry // the host's entry: what says, at every chunk, whether the host is gated now (a RENEW may have made it so since the stream began)
usage *dayUsage // the host's gated-day record
anon bool // the stream counts in the host's anonymous slice
}
// bind gives the stream its two connections. It reports false when the stream was killed before they were ready (the caller closes them).
func (l *liveStream) bind(client, dev net.Conn) bool {
l.mu.Lock()
defer l.mu.Unlock()
if l.killed != "" {
return false
}
l.client, l.dev = client, dev
return true
}
func (l *liveStream) kill(reason string) {
l.mu.Lock()
if l.killed == "" {
l.killed = reason
}
client, dev := l.client, l.dev
l.mu.Unlock()
if client != nil {
_ = client.Close()
}
if dev != nil {
_ = dev.Close()
}
}
func (l *liveStream) killedBy() string {
l.mu.Lock()
defer l.mu.Unlock()
return l.killed
}
// routeTable maps a host to the control connection of the device that owns it. The host comes ONLY from the control plane's answer; the table
// refuses to give one host to two devices and replaces a connection only by a newer one of the SAME subject. It is safe for concurrent use.
type routeTable struct {
mu sync.Mutex
byHost map[string]*hostEntry
bySub map[string]*ctrl
lim labelLimits
}
// labelLimits are what one device host accepts: how many streams (a cap on the whole, and on what sources that are not marked good may hold of it),
// and how fast new ones may begin when they come from such sources.
type labelLimits struct {
total, anon int
rate, burst float64
}
func labelLimitsOf(cfg *Config) labelLimits {
return labelLimits{total: cfg.MaxStreamsPerHost, anon: min(cfg.LabelAnonSlice, cfg.MaxStreamsPerHost), rate: float64(cfg.LabelNewPerS), burst: float64(cfg.LabelNewBurst)}
}
func newRouteTable() *routeTable {
return &routeTable{byHost: map[string]*hostEntry{}, bySub: map[string]*ctrl{}}
}
// activate binds cc to host as subject sub. It returns the connection it replaced (the same subject's older one: the caller closes it) or
// errHostTaken when another subject holds the host. bound, when set, runs under the table's lock once cc is bound, so whatever it records is
// ordered like the routing decisions are.
func (rt *routeTable) activate(cc *ctrl, sub, host string, bound func()) (*ctrl, error) {
rt.mu.Lock()
defer rt.mu.Unlock()
he := rt.byHost[host]
if he != nil && he.cc != nil && he.cc != cc && he.cc.subject() != sub {
return nil, errHostTaken
}
var old *ctrl
prevHost := ""
if prev := rt.bySub[sub]; prev != nil && prev != cc {
old, prevHost = prev, prev.hostName()
if pe := rt.byHost[prevHost]; pe != nil && pe.cc == prev {
pe.cc = nil
}
}
he = rt.byHost[host] // looked up again: unbinding the older connection must not have dropped it from under us
if he == nil {
he = &hostEntry{}
rt.byHost[host] = he
}
he.cc = cc
rt.bySub[sub] = cc
if prevHost != "" && prevHost != host {
if pe := rt.byHost[prevHost]; pe != nil {
rt.dropIfIdle(prevHost, pe)
}
}
cc.bind(sub, host)
if bound != nil {
bound()
}
return old, nil
}
// remove unbinds cc (it never touches a newer connection that replaced it).
func (rt *routeTable) remove(cc *ctrl) {
rt.mu.Lock()
defer rt.mu.Unlock()
sub, host := cc.subject(), cc.hostName()
if rt.bySub[sub] == cc {
delete(rt.bySub, sub)
}
if he := rt.byHost[host]; he != nil && he.cc == cc {
he.cc = nil
rt.dropIfIdle(host, he)
}
}
func (rt *routeTable) dropIfIdle(host string, he *hostEntry) {
if he.cc == nil && he.streams == 0 {
delete(rt.byHost, host)
}
}
func (rt *routeTable) lookup(host string) (*ctrl, *hostEntry) {
rt.mu.Lock()
defer rt.mu.Unlock()
he := rt.byHost[host]
if he == nil {
return nil, nil
}
return he.cc, he
}
// The reasons routes.acquire gives for a stream it does not admit.
const (
admitUnknown = "unknown_host" // no device is online under the name
admitHostLimit = "host_limit" // the host holds all the streams its slice allows
admitLabelRate = "label_rate" // the host is being asked for new streams faster than it accepts from sources that are not marked good
)
// acquire counts one more client stream on the host and registers it so that it can be killed. A source the device marked good (hint) may use any of
// the host's cap; any other source only the anonymous slice, which is what leaves room for the good ones when an attacker holds streams, and its new
// streams are also paced per host. It returns the connection to OPEN on, or why the stream is refused.
func (rt *routeTable) acquire(host string, hint bool, now time.Time) (*ctrl, *liveStream, string) {
rt.mu.Lock()
defer rt.mu.Unlock()
he := rt.byHost[host]
if he == nil || he.cc == nil {
return nil, nil, admitUnknown
}
if he.streams >= int64(rt.lim.total) || (!hint && he.anon >= int64(rt.lim.anon)) {
return nil, nil, admitHostLimit
}
if !hint && !he.newOnes.take(now, rt.lim.rate, rt.lim.burst) {
return nil, nil, admitLabelRate
}
he.streams++
if !hint {
he.anon++
}
ls := &liveStream{he: he, anon: !hint}
if he.live == nil {
he.live = map[*liveStream]struct{}{}
}
he.live[ls] = struct{}{}
return he.cc, ls, ""
}
func (rt *routeTable) release(host string, ls *liveStream) {
rt.mu.Lock()
defer rt.mu.Unlock()
he := ls.he
he.streams--
if ls.anon {
he.anon--
}
delete(he.live, ls)
if rt.byHost[host] == he {
rt.dropIfIdle(host, he)
}
}
// setGated records, for the host cc serves, whether the control plane last called it gated. A connection that has been replaced says nothing.
func (rt *routeTable) setGated(cc *ctrl, gated bool) {
rt.mu.Lock()
defer rt.mu.Unlock()
if he := rt.byHost[cc.hostName()]; he != nil && he.cc == cc {
he.gated.Store(gated)
}
}
// onlineCount is the number of device hosts with a control connection bound to them.
func (rt *routeTable) onlineCount() int {
rt.mu.Lock()
defer rt.mu.Unlock()
n := 0
for _, he := range rt.byHost {
if he.cc != nil {
n++
}
}
return n
}
// killStreams kills the client streams being served on host. Unless force, it does nothing while a connection of the device is bound to the host (an
// expiry that raced a newer connection must not cut what that one serves).
func (rt *routeTable) killStreams(host, reason string, force bool) int {
rt.mu.Lock()
var doomed []*liveStream
if he := rt.byHost[host]; he != nil && (force || he.cc == nil) {
for ls := range he.live {
doomed = append(doomed, ls)
}
}
rt.mu.Unlock()
for _, ls := range doomed {
ls.kill(reason)
}
return len(doomed)
}
// killAllStreams kills every client stream on every host (the kill switch, a feed reset).
func (rt *routeTable) killAllStreams(reason string) {
rt.mu.Lock()
var doomed []*liveStream
for _, he := range rt.byHost {
for ls := range he.live {
doomed = append(doomed, ls)
}
}
rt.mu.Unlock()
for _, ls := range doomed {
ls.kill(reason)
}
}
// hostHasLabel reports whether host is <label>.<something>.
func hostHasLabel(host, label string) bool {
return len(host) > len(label) && host[len(label)] == '.' && host[:len(label)] == label
}
// hostsOfLabel are the hosts the table knows (a connection bound to them, or streams still open on them) whose label is label. A host is
// <label>.<zone>, and the label is the first 20 characters: the control plane's answer is the only thing that ever put a host in the table.
func (rt *routeTable) hostsOfLabel(label string) []string {
rt.mu.Lock()
defer rt.mu.Unlock()
var hosts []string
for host := range rt.byHost {
if hostHasLabel(host, label) {
hosts = append(hosts, host)
}
}
return hosts
}
// controls are the connections bound to a host of label, and every connection when label is "".
func (rt *routeTable) controls(label string) []*ctrl {
rt.mu.Lock()
defer rt.mu.Unlock()
var out []*ctrl
for host, he := range rt.byHost {
if he.cc != nil && (label == "" || hostHasLabel(host, label)) {
out = append(out, he.cc)
}
}
return out
}
// ---------------------------------------------------------------- the control connection of one device
// ctrl is one control connection of a device: from HELLO to its close.
type ctrl struct {
s *Server
id string
conn net.Conn
wmu sync.Mutex
mu sync.Mutex
sub, host string
status string
plan string // paid or trial, as the control plane last said (the grace is for paid)
jti string // the session the device last authenticated with: what a refresh of this connection tells the control plane
authAt int64 // the control plane's own time (epoch s) at its last answer for this connection: a feed event older than it is already in it
validUntil time.Time // what the control plane granted, never changed by the node
graceUntil time.Time // a paid connection's end while the control plane does not answer (zero when it is not in its grace)
renewDown bool // the last RENEW got no answer from the control plane (a timeout, a 5xx, a 429)
refused bool // the last RENEW got an answer from the control plane, and it was no: the control plane being down later does not extend it
ready bool // an OK was sent: the connection is in the routing table
lastRenew time.Time
stop func() bool // the timer that ends the connection at validUntil (or at the end of its grace)
refreshing, refreshAgain bool // a refresh of this connection (feed pause/resume) is running, and another event came while it ran
done chan struct{}
once sync.Once
reason atomic.Value // string, why it was closed
}
func (s *Server) newCtrl(c net.Conn) *ctrl {
cc := &ctrl{s: s, id: fmt.Sprintf("c%d", s.connSeq.Add(1)), conn: c, done: make(chan struct{})}
s.mu.Lock()
s.conns[cc] = struct{}{}
s.mu.Unlock()
return cc
}
func (c *ctrl) subject() string { c.mu.Lock(); defer c.mu.Unlock(); return c.sub }
func (c *ctrl) hostName() string { c.mu.Lock(); defer c.mu.Unlock(); return c.host }
func (c *ctrl) bind(sub, host string) {
c.mu.Lock()
c.sub, c.host = sub, host
c.mu.Unlock()
}
// setReady records what the control plane decided: the connection serves streams from now on, until validUntil. A new decision ends any grace.
func (c *ctrl) setReady(resp AuthorizeResponse, jti string, validUntil time.Time) {
c.mu.Lock()
c.ready, c.status, c.plan, c.jti, c.authAt, c.validUntil = true, resp.Status, resp.Plan, jti, resp.ServerTime, validUntil
c.graceUntil, c.renewDown, c.refused = time.Time{}, false, false
c.mu.Unlock()
}
// setStatus records a status the control plane gave in an answer that does not extend the connection (a refresh after a pause or a resume).
func (c *ctrl) setStatus(resp AuthorizeResponse) {
c.mu.Lock()
c.status, c.plan, c.authAt = resp.Status, resp.Plan, resp.ServerTime
c.mu.Unlock()
}
// pause stops this connection from opening streams until the control plane says otherwise.
func (c *ctrl) pause() {
c.mu.Lock()
c.status = statusPaused
c.mu.Unlock()
}
// markRenew records how the control plane came out of a RENEW that was not granted: it did not answer (down), or it answered no (refused).
func (c *ctrl) markRenew(down, refused bool) {
c.mu.Lock()
c.renewDown, c.refused = down, refused
c.mu.Unlock()
}
// canOpen reports whether a client stream may be opened on this connection at now: an OK was sent, the device is not paused and valid_until has not
// come. A paused connection stays up (a resume arrives as a statement) but opens nothing.
func (c *ctrl) canOpen(now time.Time) bool {
c.mu.Lock()
defer c.mu.Unlock()
return c.ready && c.status != statusPaused && now.Before(c.validUntil)
}
// tooSoon records a RENEW at now and reports whether the previous one was less than gap ago.
func (c *ctrl) tooSoon(now time.Time, gap time.Duration) bool {
c.mu.Lock()
defer c.mu.Unlock()
if !c.lastRenew.IsZero() && now.Sub(c.lastRenew) < gap {
return true
}
c.lastRenew = now
return false
}
func (c *ctrl) isReady() bool { c.mu.Lock(); defer c.mu.Unlock(); return c.ready }
func (c *ctrl) writeLocked(line string, at time.Time) error {
err := c.sendLocked(line)
c.s.emit(at, map[string]any{"conn": c.id, "ev": "send", "line": line})
return err
}
// sendLocked puts one line on the wire without recording it, for a caller that records it itself.
func (c *ctrl) sendLocked(line string) error {
_ = c.conn.SetWriteDeadline(time.Now().Add(c.s.cfg.WriteTimeout))
_, err := io.WriteString(c.conn, line+"\n")
return err
}
func (c *ctrl) write(line string) error { return c.writeAt(line, c.s.now()) }
// writeAt writes one line; at is the moment recorded in the trace (the moment the decision to write it was taken).
func (c *ctrl) writeAt(line string, at time.Time) error {
c.wmu.Lock()
defer c.wmu.Unlock()
return c.writeLocked(line, at)
}
// close ends the connection for reason (a CLOSE reason, or another word for the log). A reason the device is told (CLOSE <reason>) is sent first,
// unless the writer is stuck.
func (c *ctrl) close(reason string) {
c.once.Do(func() {
c.reason.Store(reason)
close(c.done)
c.mu.Lock()
if c.stop != nil {
c.stop()
}
c.mu.Unlock()
// The CLOSE line goes out under the write lock, so it follows an OK that is still being written; a writer that is stuck is not waited for.
told, ready := false, false
if c.lockWrite(closeWait) {
if ready = c.isReady(); ready && inList(policyData.Protocol.CloseReasons, reason) {
_ = c.writeLocked(closeLine(reason), c.s.now())
told = true
}
c.wmu.Unlock()
} else {
ready = c.isReady()
}
if ready && !told {
// The transport ended without a CLOSE line (the device went away, the write was stuck). The recording still has to know this
// connection is over, so a trace gets a marker that never went on the wire.
c.s.emit(c.s.now(), map[string]any{"conn": c.id, "ev": "send", "line": "CLOSE", "wire": false})
}
_ = c.conn.Close()
host := c.hostName()
c.s.routes.remove(c)
c.s.mu.Lock()
delete(c.s.conns, c)
c.s.mu.Unlock()
// A connection that ends because its grant ended (not because the device went away) takes the streams of its host with it, unless a newer
// connection of the device already serves them.
if ready && endsStreams(reason) {
c.s.routes.killStreams(host, reason, false)
}
})
}
// endsStreams says whether a connection closed for reason takes the host's open streams with it: the grant ended (expired, revoked, released) or the
// kill switch is on. A device that went away, a replaced connection and a node going down leave them to their own limits.
func endsStreams(reason string) bool {
switch reason {
case closeExpired, closeRevoked, closeReleased, closeKilled:
return true
}
return false
}
// closeWait is how long a close waits for a writer that holds the connection's write lock before it gives up on the CLOSE line.
const closeWait = 250 * time.Millisecond
func (c *ctrl) lockWrite(wait time.Duration) bool {
deadline := time.Now().Add(wait)
for !c.wmu.TryLock() {
if time.Now().After(deadline) {
return false
}
time.Sleep(time.Millisecond)
}
return true
}
func (c *ctrl) closeReason() string {
if r, ok := c.reason.Load().(string); ok {
return r
}
return ""
}
// arm schedules the end of the connection at validUntil (CLOSE expired); a RENEW arms it again for the new moment.
func (c *ctrl) arm(validUntil time.Time) {
c.mu.Lock()
defer c.mu.Unlock()
if c.stop != nil {
c.stop()
}
c.stop = c.s.clk.AfterFunc(validUntil.Sub(c.s.now()), func() { c.expire(validUntil) })
}
// expire is the end of the grant the timer was armed for. A grant a RENEW replaced since is not this one, and nothing happens. Otherwise the
// connection closes, except for a paid one while the control plane does not answer: it keeps its connection and its streams (but opens no new
// stream: canOpen is false from valid_until) for the grace, and closes when that is over. A trial or a gated connection is never extended, and
// neither is a paid one the control plane did answer (its RENEW was refused, or never came).
func (c *ctrl) expire(validUntil time.Time) {
c.mu.Lock()
current := c.validUntil.Equal(validUntil)
inGrace := !c.graceUntil.IsZero()
paid := c.plan == "paid"
down := c.renewDown
refused := c.refused
c.mu.Unlock()
if !current {
return
}
now := c.s.now()
if !inGrace && paid && !refused && c.s.cfg.Grace > 0 && (down || c.s.health.down(now, c.s.cfg.DownQuiet)) {
graceUntil := validUntil.Add(c.s.cfg.Grace)
c.mu.Lock()
c.graceUntil = graceUntil
if c.stop != nil {
c.stop()
}
c.stop = c.s.clk.AfterFunc(graceUntil.Sub(now), func() { c.expire(validUntil) })
c.mu.Unlock()
c.s.log.Warn("control_grace", "conn", c.id, "grace_s", int64(c.s.cfg.Grace/time.Second))
return
}
c.close(closeExpired)
}
// bufConn serves reads through the bufio.Reader that parsed the first protocol line, so no byte the peer sent right after it is lost.
type bufConn struct {
net.Conn
r *bufio.Reader
}
func (b *bufConn) Read(p []byte) (int, error) { return b.r.Read(p) }
func (b *bufConn) CloseWrite() error {
if cw, ok := b.Conn.(interface{ CloseWrite() error }); ok {
return cw.CloseWrite()
}
return b.Conn.Close()
}
// prefixConn replays the bytes already read from a connection before reading on: the ClientHello that was peeked to learn the SNI.
type prefixConn struct {
net.Conn
prefix []byte
}
func (p *prefixConn) Read(b []byte) (int, error) {
if len(p.prefix) > 0 {
n := copy(b, p.prefix)
p.prefix = p.prefix[n:]
return n, nil
}
return p.Conn.Read(b)
}
// ---------------------------------------------------------------- :443
// sourceOf is the source address of an accepted connection and whether it is in the OpenAI list.
func (s *Server) sourceOf(c net.Conn) (peer netip.Addr, ok, openai bool) {
if s.cfg.peerOf != nil {
peer, ok = s.cfg.peerOf(c)
} else {
peer, ok = peerAddr(c)
}
return peer, ok, ok && s.openai.contains(peer)
}
// short is the per-boot salted short hash of a word (an address, a name): how a log line tells two of them apart without holding either. The salt
// lives in memory only, so the hash cannot be read back, and it changes at every start, so it cannot be followed from one boot to the next.
func (s *Server) short(word string) string {
return fmt.Sprintf("%08x", uint32(maphash.String(s.seed, word)))
}
// refused counts a connection or stream the node did not serve and writes its log line, at most one a second per reason: a flood must not become a flood
// of log lines (the count is in the metrics, and the line says how many were left out). The line never names a source or a name.
func (s *Server) refused(level slog.Level, msg, reason string, attrs ...any) {
s.met.rejected.inc(reason)
if ok, cut := s.sampler.allow(msg + "/" + reason); ok {
attrs = append(attrs, "reason", reason)
if cut > 0 {
attrs = append(attrs, "suppressed", cut)
}
s.log.Log(context.Background(), level, msg, attrs...)
}
}
// handleHTTPS is the one public listener. It reads only the cleartext ClientHello for the SNI: the node's own name is the control handler (the
// device speaks protocol v2 to it over TLS with ALPN all-tunnel/2), any other name is a client of a device and is spliced as raw bytes.
//
// Before a byte is read, the source is admitted: a source that is banned, that opens connections faster than the policy lets it, or that already holds
// the most it may is dropped without an answer (limits.go).
func (s *Server) handleHTTPS(c net.Conn) {
defer s.wg.Done()
handedOff := false // an ATTACHed data connection belongs to its stream from then on
defer func() {
if !handedOff {
_ = c.Close()
}
}()
id := newStreamID()
start := s.now()
peer, havePeer, fromOpenAI := s.sourceOf(c)
slotDone := func() {}
if havePeer {
rel, why := s.ips.admit(peer, fromOpenAI)
if why != "" {
s.refused(slog.LevelInfo, "conn_reject", why, "stream", id)
return
}
slotDone = rel
}
defer slotDone()
// The unauthenticated budget has a reserve, as the stream budget has: a source that is neither an address of the OpenAI list, nor one that holds (or just held) a device's
// control connection, nor one a device marked good, may not take the share kept for those (policy node_limits.openai_reserved_pct). Twenty anonymous addresses holding the
// most a source may hold fill the anonymous share, never the whole budget: ChatGPT's clients and the devices that reconnect (they are unauthenticated until their PROOF) go on.
anon := !(havePeer && (fromOpenAI || s.ips.knownDevice(peer) || s.hints.goodAnywhere(peer)))
if anon && !tryAcquire(&s.anonUnauth, s.anonUnauthMax()) {
s.refused(slog.LevelWarn, "conn_reject", "unauth_anon_limit")
return
}
if !tryAcquire(&s.unauth, int64(s.cfg.MaxUnauth)) {
if anon {
s.anonUnauth.Add(-1)
}
s.refused(slog.LevelWarn, "conn_reject", "unauth_limit")
return
}
released := false
release := func() {
if !released {
released = true
s.unauth.Add(-1)
if anon {
s.anonUnauth.Add(-1)
}
}
}
defer release()
var unmark func()
// strike is a source that sent something no client of a device sends: a name that is not a device online, or a ClientHello that is no TLS. Too
// many different ones and the source is dropped at accept for a while (an address of the OpenAI list is never struck: ChatGPT retrying an
// offline device is not a scan).
strike := func(name string) {
// An address a device marked good (it just completed authenticated calls) is not a scanner, whatever names it asks for: at fleet scale a shared
// egress address asks for many devices that are offline at that moment.
if havePeer && !s.hints.goodAnywhere(peer) && s.ips.strike(peer, name, fromOpenAI) {
s.met.bansTotal.Add(1)
s.log.Info("source_banned", "source", s.short(peer.String()), "ban_s", int64(s.cfg.UnknownSNIBan/time.Second))
}
}
peeked, sni, err := readClientHello(c, s.cfg.HelloTimeout)
if err != nil && !errors.Is(err, errBadSNI) {
// A ClientHello that is truncated, malformed, too large or never finished never produces an OPEN.
s.emit(start, map[string]any{"ev": "client_hello", "sni": "", "complete": false})
reason := helloReason(err)
s.refused(slog.LevelInfo, "conn_reject", reason, "stream", id, "peeked_bytes", len(peeked))
if reason == "not_tls" || reason == "bad_hello" || reason == "hello_too_large" {
strike("")
}
return
}
if err != nil || sni == "" {
reason := "no_sni"
if err != nil {
reason = "bad_sni"
}
s.emit(s.now(), map[string]any{"ev": "tls_alert", "sni": ""})
s.refused(slog.LevelInfo, "conn_reject", reason, "stream", id, "peeked_bytes", len(peeked))
strike("")
sendAlert(c, alertUnrecognizedName)
return
}
if sni == s.cfg.NodeHost {
// A connection of the node's own name that proves who it is (a device's control connection after its proof, a data connection after its ATTACH)
// stops being an unknown source's: it gives back its slot and the token it took, and a control connection marks its address as a device's for as
// long as it lives, so that the connections of the streams it serves are not held to what an anonymous source gets (limits.go).
defer func() {
if unmark != nil {
unmark()
}
}()
handedOff = s.serveOwnName(c, peeked, func(control bool) {
release()
if havePeer {
s.ips.refund(peer, fromOpenAI)
if control && unmark == nil {
unmark = s.ips.markDevice(peer)
}
}
slotDone()
})
return
}
release()
s.met.observeClientHello(peeked)
s.serveClient(c, id, sni, peeked, start, peer, havePeer, strike)
}
// classOf names the class of a client stream's source for the OPEN: the configured classifier when there is one, otherwise openai for an address of
// the OpenAI list, hint for one the device marked, anon for the rest. marked says whether the device marked the address (what puts the stream in the
// host's hint slice, whatever class the OPEN says).
func (s *Server) classOf(peer netip.Addr, havePeer bool, host string) (class string, marked bool) {
class = classAnon
if havePeer {
marked = s.hints.good(host, peer)
}
switch {
case s.cfg.Classify != nil && havePeer:
if got := s.cfg.Classify(peer, host); inList(policyData.Protocol.SourceClasses, got) {
class = got
}
case havePeer && s.openai.contains(peer):
class = classOpenAI
case marked:
class = classHint
}
return class, marked || class == classHint
}
// serveClient splices one client of a device. Everything the node learned is the SNI and the ClientHello's bytes, and the OPEN is sent only now
// that the ClientHello is complete and well formed.
func (s *Server) serveClient(c net.Conn, id, sni string, peeked []byte, start time.Time, peer netip.Addr, havePeer bool, strike func(name string)) {
if s.cfg.divert != nil && s.cfg.divert(c, peeked, s.log) {
return
}
class, marked := s.classOf(peer, havePeer, sni)
peerText := "0.0.0.0"
if havePeer {
peerText = peer.String()
}
hello := map[string]any{"ev": "client_hello", "sni": sni, "complete": true, "peer_ip": peerText, "class": class}
s.emit(s.now(), hello)
if s.draining.Load() {
// A draining node takes no new client: the TLS alert, never an OPEN. (The client's ClientHello is not forwarded anywhere.)
s.refused(slog.LevelInfo, "stream_reject", "draining", "stream", id, "peeked_bytes", len(peeked))
s.alert(c, sni, alertInternalError)
return
}
// The global budget: every stream counts in it, and a stream whose source is neither ChatGPT's (the OpenAI list) nor marked good by a device may not
// take the share that is reserved for those (policy node_limits.openai_reserved_pct).
if !tryAcquire(&s.streams, int64(s.cfg.MaxStreams)) {
s.refused(slog.LevelWarn, "stream_reject", "global_limit", "stream", id)
s.alert(c, sni, alertInternalError)
return
}
defer s.streams.Add(-1)
if class == classAnon && !marked {
if !tryAcquire(&s.anonStreams, int64(s.cfg.MaxStreams-s.cfg.MaxStreams*s.cfg.OpenAIReservedPct/100)) {
s.refused(slog.LevelWarn, "stream_reject", "anon_global_limit", "stream", id)
s.alert(c, sni, alertInternalError)
return
}
defer s.anonStreams.Add(-1)
}
cc, live, why := s.routes.acquire(sni, marked, s.now())
if why != "" {
desc := byte(alertInternalError)
if why == admitUnknown {
desc = alertUnrecognizedName
strike(sni)
}
s.refused(slog.LevelInfo, "stream_reject", why, "stream", id, "peeked_bytes", len(peeked))
s.alert(c, sni, desc)
return
}
defer s.routes.release(sni, live)
// The host's record of today's gated budget: every stream holds it, so that a host that turns gated while the stream is open is held to it too.
live.usage = s.gates.acquire(sni)
defer s.gates.release(live.usage)
if live.he.gated.Load() {
if ok, reason := live.usage.admit(s.now(), s.cfg.GatedBytesPerDay, s.cfg.GatedStreamsPerDay); !ok {
s.refused(slog.LevelInfo, "stream_reject", reason, "stream", id, "peeked_bytes", len(peeked))
s.alert(c, sni, alertInternalError)
return
}
}
dc, err := s.openStream(cc, id, peerText, class)
if err != nil {
s.refused(slog.LevelInfo, "stream_reject", err.Error(), "stream", id, "peeked_bytes", len(peeked), "wait_ms", s.now().Sub(start).Milliseconds())
s.alert(c, sni, alertInternalError)
return
}
defer dc.Close()
if !live.bind(c, dc) {
// Cut (a revoke, a pause) between the OPEN and the ATTACH: nothing of the client's goes to the device.
s.refused(slog.LevelInfo, "stream_reject", live.killedBy(), "stream", id, "peeked_bytes", len(peeked))
s.alert(c, sni, alertInternalError)
return
}
s.met.streamsOpened.Add(1)
s.log.Info("stream_open", "stream", id, "hello_bytes", len(peeked), "setup_ms", s.now().Sub(start).Milliseconds(), "class", class)
res := s.splice(id, c, dc, peeked, live)
if why := live.killedBy(); why != "" {
res.reason = why
}
s.met.streamEnd.inc(res.reason)
s.log.Info("stream_close", "stream", id, "up_bytes", res.up, "down_bytes", res.down,
"duration_ms", s.now().Sub(start).Milliseconds(), "reason", res.reason)
}
func (s *Server) alert(c net.Conn, sni string, desc byte) {
s.emit(s.now(), map[string]any{"ev": "tls_alert", "sni": sni})
sendAlert(c, desc)
}
// openStream registers a pending stream, tells the device (OPEN over its control connection) and waits for its ATTACH. The returned connection has
// had its ATTACH line consumed; the caller's first write is the first byte of the raw stream (there is deliberately no reply line in between, so a
// device can hand the connection straight to its TLS layer).
func (s *Server) openStream(cc *ctrl, id, peer, class string) (net.Conn, error) {
at := s.now()
if !cc.canOpen(at) {
return nil, errOffline
}
secret := newSecret()
p := &pendingStream{cc: cc, secret: secret, ch: make(chan net.Conn, 1)}
s.mu.Lock()
s.pending[id] = p
s.mu.Unlock()
if err := cc.writeAt(formatLine(message{"verb": "OPEN", "stream_id": id, "secret": secret, "peer_ip": peer, "class": class}), at); err != nil {
s.dropPending(id)
cc.close("write_error")
return nil, errControlWrite
}
timer := time.NewTimer(s.cfg.AttachTimeout)
defer timer.Stop()
done := cc.done
wait:
for {
select {
case dc := <-p.ch:
return dc, nil
case <-timer.C:
break wait
case <-done:
if cc.closeReason() != closeReplaced {
break wait
}
// A newer connection of the same device took over: this OPEN was sent with the old one, and its attach secret is still good.
done = nil
case <-s.ctx.Done():
break wait
}
}
// Failed: unregister, but an ATTACH may have just landed.
if !s.dropPending(id) {
return <-p.ch, nil
}
switch {
case s.ctx.Err() != nil:
return nil, errShuttingDown
case cc.closeReason() != "" && cc.closeReason() != closeReplaced:
return nil, errOffline
}
return nil, errAttachTimeout
}
// dropPending removes id; it reports whether the entry was still there.
func (s *Server) dropPending(id string) bool {
s.mu.Lock()
defer s.mu.Unlock()
_, ok := s.pending[id]
delete(s.pending, id)
return ok
}
// ---------------------------------------------------------------- splice
type spliceResult struct {
up, down int64 // client->device, device->client
reason string
}
func classifyErr(err error) string {
switch {
case err == nil:
return "closed"
case errors.Is(err, errByteCap):
return "byte_cap"
case errors.Is(err, errGatedBytes):
return "gated_bytes"
case errors.Is(err, os.ErrDeadlineExceeded):
return "idle"
default:
return "conn_error"
}
}
func closeWrite(c net.Conn) {
if cw, ok := c.(interface{ CloseWrite() error }); ok {
_ = cw.CloseWrite()
return
}
_ = c.Close()
}
// pump copies src to dst with per-read idle deadlines and the shared byte budget. A clean EOF half-closes dst and returns nil. While the stream's host
// is gated, every chunk is also charged to the host's budget of the day, and the stream ends where the budget does. watch, when set, reads the
// device's first bytes for the handshake observer.
func (s *Server) pump(id, dir string, dst, src net.Conn, ctr, total *atomic.Int64, live *liveStream, watch *serverHelloWatch) error {
// A pump spends the whole life of an idle stream blocked in a read, so the buffer it waits with is a small one (4 KiB); it takes a 32 KiB one only for a read that filled
// the small one (a bulk transfer) and gives it back when a read comes back short. An idle stream then holds about 8 KiB of buffer instead of 64 KiB (the acceptance is under 64 KiB of memory per stream).
sp := s.smallBufPool.Get().(*[]byte)
small := *sp
defer s.smallBufPool.Put(sp)
var bp *[]byte // the big buffer, while this pump holds one
giveBack := func() {
if bp != nil {
s.bigBuffersInUse.Add(-1)
s.bufPool.Put(bp)
bp = nil
}
}
defer giveBack()
buf := small
for {
_ = src.SetReadDeadline(time.Now().Add(s.cfg.StreamIdle))
n, rerr := src.Read(buf)
if n > 0 {
chunk, spent := buf[:n], false
if live != nil && live.he.gated.Load() {
if got := live.usage.charge(s.now(), int64(n), s.cfg.GatedBytesPerDay); got < int64(n) {
chunk, spent = buf[:got], true
}
}
if total.Add(int64(len(chunk))) > s.cfg.MaxStreamBytes {
return errByteCap
}
if len(chunk) > 0 {
s.tapBytes(id, dir, chunk)
if watch != nil {
watch.feed(chunk)
}
_ = dst.SetWriteDeadline(time.Now().Add(s.cfg.StreamIdle))
if _, werr := dst.Write(chunk); werr != nil {
return werr
}
ctr.Add(int64(len(chunk)))
s.met.bytes.add(dir, int64(len(chunk)))
}
if spent {
s.met.gatedCut.Add(1)
return errGatedBytes
}
}
if rerr != nil {
if rerr == io.EOF {
closeWrite(dst)
return nil
}
return rerr
}
switch { // the buffer for the next read: bulk traffic gets the big one, a trickle (and the wait of an idle stream) the small one
case bp == nil && n == len(small):
bp = s.bufPool.Get().(*[]byte)
s.bigBuffersInUse.Add(1)
buf = *bp
case bp != nil && n < len(small):
giveBack()
buf = small
}
}
}
// splice copies bytes both ways between client and dev until both sides are done, one fails, or a limit trips. preload (the peeked ClientHello) is
// written to dev before anything else from the client. Neither side's bytes are interpreted.
func (s *Server) splice(id string, client, dev net.Conn, preload []byte, live *liveStream) spliceResult {
var up, down, total atomic.Int64
var rmu sync.Mutex
reason := ""
setReason := func(r string) {
rmu.Lock()
if reason == "" {
reason = r
}
rmu.Unlock()
}
closeBoth := func() { _ = client.Close(); _ = dev.Close() }
age := time.AfterFunc(s.cfg.StreamMaxAge, func() { setReason("max_age"); closeBoth() })
defer age.Stop()
stopWatch := make(chan struct{})
defer close(stopWatch)
go func() {
select {
case <-s.ctx.Done():
setReason("shutdown")
closeBoth()
case <-stopWatch:
}
}()
results := make(chan error, 2)
go func() { // client -> device
if len(preload) > 0 {
if live.he.gated.Load() && live.usage.charge(s.now(), int64(len(preload)), s.cfg.GatedBytesPerDay) < int64(len(preload)) {
// a ClientHello cut short is of no use to anyone: nothing of it goes to the device
s.met.gatedCut.Add(1)
results <- errGatedBytes
return
}
_ = dev.SetWriteDeadline(time.Now().Add(s.cfg.WriteTimeout))
if _, err := dev.Write(preload); err != nil {
results <- err
return
}
s.tapBytes(id, "up", preload)
up.Add(int64(len(preload)))
total.Add(int64(len(preload)))
s.met.bytes.add("up", int64(len(preload)))
}
results <- s.pump(id, "up", dev, client, &up, &total, live, nil)
}()
go func() { // device -> client
results <- s.pump(id, "down", client, dev, &down, &total, live, newServerHelloWatch(s.met))
}()
first := <-results
if first == nil {
// One direction ended cleanly (and was half-closed): the other gets a bounded time to finish.
t := time.NewTimer(s.cfg.HalfCloseTimeout)
select {
case second := <-results:
setReason(classifyErr(second))
case <-t.C:
setReason("half_close_timeout")
closeBoth()
<-results
}
t.Stop()
} else {
setReason(classifyErr(first))
closeBoth()
<-results
}
closeBoth()
rmu.Lock()
defer rmu.Unlock()
return spliceResult{up: up.Load(), down: down.Load(), reason: reason}
}
// ---------------------------------------------------------------- the node's own name: control and data connections of devices
func readLine(br *bufio.Reader) (string, error) {
b, err := br.ReadSlice('\n')
if err != nil {
return "", err
}
return string(b[:len(b)-1]), nil
}
func writeLineThenClose(c net.Conn, line string) {
_ = c.SetWriteDeadline(time.Now().Add(2 * time.Second))
_, _ = io.WriteString(c, line+"\n")
_ = c.Close()
}
// serveOwnName terminates the DEVICE-facing TLS on the node's own name (its public-CA certificate, TLS 1.3, ALPN all-tunnel/2), reads the first line
// and dispatches HELLO or ATTACH. This is the tunnel's own connection to its own client, never a client's TLS. It reports whether the connection was
// handed to a stream (an ATTACH that matched), in which case the caller must not close it.
func (s *Server) serveOwnName(raw net.Conn, peeked []byte, release func(control bool)) bool {
tc := tls.Server(&prefixConn{Conn: raw, prefix: peeked}, s.tlsCfg)
_ = tc.SetDeadline(time.Now().Add(s.cfg.FirstLineTimeout))
if err := tc.Handshake(); err != nil {
s.log.Debug("control_tls_failed")
return false
}
if tc.ConnectionState().NegotiatedProtocol != policyData.Protocol.ALPN {
s.log.Info("control_reject", "reason", "bad_alpn")
return false
}
br := bufio.NewReaderSize(tc, policyData.Protocol.LineMaxBytes)
line, err := readLine(br)
if err != nil {
if errors.Is(err, bufio.ErrBufferFull) {
writeLineThenClose(tc, errLine(codeLineTooLong))
} else {
s.log.Info("control_reject", "reason", "bad_first_line")
}
return false
}
first, perr := parseLine(line, dirD2N)
if perr != nil {
// An ATTACH or a HELLO with a bad field answers with its own word; anything else is not a command.
code := errCode(perr)
if v, _, _ := strings.Cut(line, " "); v != "HELLO" && v != "ATTACH" {
code = codeBadCommand // contract section 2: a first line that is not HELLO or ATTACH is bad_command, whatever else is wrong with it
}
s.log.Info("control_reject", "reason", code)
writeLineThenClose(tc, errLine(code))
return false
}
switch first.str("verb") {
case "HELLO":
cc := s.newCtrl(tc)
s.emit(s.now(), map[string]any{"ev": "recv", "conn": cc.id, "line": line})
if s.draining.Load() {
// A draining node takes no new device either: it is told to try the other node. (An ATTACH for a stream that already has its OPEN is served.)
s.log.Info("control_reject", "conn", cc.id, "reason", "draining")
_ = cc.write(errLine(codeControlUnavailable))
cc.close("refused")
return false
}
s.serveControl(cc, br, first, release)
return false
case "ATTACH":
return s.serveAttach(tc, br, first, line, fmt.Sprintf("d%d", s.connSeq.Add(1)), release)
}
s.log.Info("control_reject", "reason", codeBadCommand)
writeLineThenClose(tc, errLine(codeBadCommand))
return false
}
// exchange is one authentication in flight on a control connection: the HELLO that opened it, or a RENEW.
type exchange struct {
kind string // "hello" or "renew"
token string
session *SessionClaims
nonce string
deadline time.Time
}
// refuse answers a failed HELLO or RENEW (or PROOF) with ERR. A refused HELLO closes the connection; a refused RENEW keeps it, with its old
// valid_until.
func (s *Server) refuse(cc *ctrl, kind, code string) bool {
_ = cc.write(errLine(code))
if kind == "hello" {
cc.close("refused")
return false
}
return true
}
// beginExchange verifies the session of a HELLO or RENEW and, when it holds, sends the CHALLENGE. A RENEW must be for the same subject as the
// connection. It reports whether the connection goes on.
func (s *Server) beginExchange(cc *ctrl, ex *exchange, kind, token string) bool {
*ex = exchange{}
now := s.now()
session, err := verifySession(token, now, s.cfg.Keys, s.cfg.ClockSkew)
if err != nil {
code := errFor(tokenWord(err), codeAuth)
if kh, ok := s.cfg.Keys.(keyHealth); ok && tokenWord(err) == tokUnknownKID && !kh.Healthy() {
// The node cannot vouch for the control plane's keys (stale beyond the policy's window): not the device's fault, so not auth.
code = codeControlUnavailable
if kind == "renew" {
// ... and it is the control plane being unreachable, which is what a paid device's grace is for: without this the connection closed at valid_until although the
// device did everything right (its session was refused only because the node could not read the control plane's keys).
cc.markRenew(true, false)
}
}
s.log.Info("session_refused", "conn", cc.id, "kind", kind, "reason", tokenWord(err), "answer", code)
return s.refuse(cc, kind, code)
}
if kind == "renew" && session.Subject != cc.subject() {
s.log.Info("session_refused", "conn", cc.id, "kind", kind, "reason", "other_subject")
return s.refuse(cc, kind, codeAuth)
}
*ex = exchange{kind: kind, token: token, session: session, nonce: newNonce(), deadline: now.Add(s.cfg.FirstLineTimeout)}
if err := cc.write(formatLine(message{"verb": "CHALLENGE", "nonce": ex.nonce})); err != nil {
cc.close("write_error")
return false
}
return true
}
// serveControl runs a control connection from its HELLO to its end. release gives back the unauthenticated slot once the device has authenticated.
func (s *Server) serveControl(cc *ctrl, br *bufio.Reader, hello message, release func(control bool)) {
tc := cc.conn
defer cc.close("closed")
defer s.routes.remove(cc) // idempotent: a close that raced an activation must not leave a route behind
start := s.now()
ex := &exchange{}
if !s.beginExchange(cc, ex, "hello", hello.str("session")) {
return
}
reason := ""
for {
deadline := s.now().Add(s.cfg.DeadPeer)
if !cc.isReady() || ex.nonce != "" {
deadline = s.now().Add(s.cfg.FirstLineTimeout)
if !ex.deadline.IsZero() && ex.deadline.Before(deadline) {
deadline = ex.deadline
}
}
_ = tc.SetReadDeadline(deadline)
line, err := readLine(br)
if err != nil {
switch {
case errors.Is(err, bufio.ErrBufferFull):
_ = cc.write(errLine(codeLineTooLong))
reason = closeProtocol
case errors.Is(err, os.ErrDeadlineExceeded) && ex.nonce != "" && cc.isReady():
// The device never answered a RENEW's CHALLENGE: the exchange is dropped, the connection and its old valid_until stay.
*ex = exchange{}
continue
case errors.Is(err, os.ErrDeadlineExceeded):
reason = closeIdle
default:
reason = "closed"
}
cc.close(reason)
s.log.Info("control_down", "conn", cc.id, "duration_ms", s.now().Sub(start).Milliseconds(), "reason", cc.closeReason())
return
}
s.emit(s.now(), map[string]any{"ev": "recv", "conn": cc.id, "line": line})
m, perr := parseLine(line, dirD2N)
if perr != nil {
_ = cc.write(errLine(errCode(perr)))
cc.close(closeProtocol)
return
}
switch m.str("verb") {
case "PROOF":
if ex.nonce == "" {
_ = cc.write(errLine(codeBadCommand))
cc.close(closeProtocol)
return
}
if !s.finishExchange(cc, ex, m.str("proof"), release) {
return
}
case "RENEW":
if !cc.isReady() || ex.nonce != "" {
_ = cc.write(errLine(codeBadCommand))
cc.close(closeProtocol)
return
}
if cc.tooSoon(s.now(), s.cfg.MinRenewGap) {
// a device cannot turn one connection into a stream of authorize calls: the connection and its old valid_until stay
if err := cc.write(errLine(codeBusy)); err != nil {
cc.close("write_error")
return
}
continue
}
if !s.beginExchange(cc, ex, "renew", m.str("session")) {
return
}
case "PING":
if err := cc.write(formatLine(message{"verb": "PONG"})); err != nil {
cc.close("write_error")
return
}
case "HINT":
if cc.isReady() {
var addrs []netip.Addr
for _, a := range m["addresses"].([]any) {
if addr, err := netip.ParseAddr(a.(string)); err == nil {
addrs = append(addrs, addr)
}
}
// filed under the host the control plane gave THIS connection: a device can mark addresses for no other host than its own
s.hints.mark(cc.hostName(), addrs)
s.met.hintsMarked.Add(1)
if s.cfg.OnHint != nil {
s.cfg.OnHint(cc.hostName(), addrs)
}
}
case "CLOSE":
cc.close("device_close")
s.log.Info("control_down", "conn", cc.id, "duration_ms", s.now().Sub(start).Milliseconds(), "reason", "device_close")
return
default: // HELLO again, or anything else the device must not send here
_ = cc.write(errLine(codeBadCommand))
cc.close(closeProtocol)
return
}
}
}
// finishExchange checks the PROOF of the exchange in flight and, when it holds, asks the control plane what the device may do and answers OK. It
// reports whether the connection goes on.
func (s *Server) finishExchange(cc *ctrl, ex *exchange, proof string, release func(control bool)) bool {
cur := *ex
*ex = exchange{} // the nonce is spent whatever the verdict
now := s.now()
pc, err := verifyProof(proof, now, cur.token, cur.session, cur.nonce, s.nodeURL, s.cfg.ClockSkew)
if err == nil && !s.replay.spend(pc.JTI, pc.IssuedAt, now, s.cfg.ClockSkew) {
err = tokFail(tokReplayedJTI)
}
if err != nil {
s.log.Info("proof_refused", "conn", cc.id, "kind", cur.kind, "reason", tokenWord(err))
return s.refuse(cc, cur.kind, errFor(tokenWord(err), codeBadProof))
}
// The device has proven itself, so it no longer holds a pre-authentication slot: a slow control plane must not starve the ClientHellos of
// clients and the ATTACHes of devices that are already connected. What waits for authorize is bounded on its own.
if cur.kind == "hello" {
release(true)
}
if s.killed.Load() {
// The kill switch is on: nobody is served, and nothing is asked of the control plane until it is cleared.
s.log.Info("authorize_skipped", "conn", cc.id, "kind", cur.kind, "reason", "kill_switch")
return s.refuse(cc, cur.kind, codeControlUnavailable)
}
// The control plane decides; nothing the device said names a host. Until its answer is acted on the feed is listened to: an event the feed
// applies while the call is in flight is not in the answer, and cannot find a connection that is not yet in the routing table.
watch := s.watchFeed()
defer watch.stop()
req := AuthorizeRequest{DeviceID: cur.session.Subject, SessionJTI: cur.session.JTI, Reason: cur.kind, Node: s.cfg.NodeID}
s.emit(s.now(), map[string]any{"conn": cc.id, "ev": "cp", "call": "authorize", "body": req.toMap()})
resp, err := s.authorize(req)
if err == nil {
if cerr := checkAuthorize(resp, req); cerr != nil {
s.log.Warn("authorize_answer_refused", "conn", cc.id, "reason", cerr.Error())
s.health.bad(s.now())
err = &ControlError{Status: 502, Code: "bad_response"}
}
}
if err != nil {
status, word := 503, "control_unavailable"
if ce, ok := err.(*ControlError); ok {
status, word = ce.Status, ce.Code
}
s.emit(s.now(), map[string]any{"conn": cc.id, "ev": "cp_reply", "status": status, "body": map[string]any{"ok": false, "error": word}})
code := errForControl(err)
if cur.kind == "renew" && !isLocalFailure(err) {
// A RENEW this node could not even send (its own queue was full or its turn did not come) says nothing about the control plane.
cc.markRenew(code == codeControlUnavailable, code != codeControlUnavailable)
}
s.log.Info("authorize_refused", "conn", cc.id, "kind", cur.kind, "reason", code)
return s.refuse(cc, cur.kind, code)
}
s.emit(s.now(), map[string]any{"conn": cc.id, "ev": "cp_reply", "status": 200, "body": resp.toMap()})
if resp.E2EKill {
// The device is told first (the switch closes every connection, this one included), then nobody is served until it is cleared.
s.log.Warn("kill_switch_seen", "conn", cc.id, "via", "authorize")
s.refuse(cc, cur.kind, codeControlUnavailable)
s.setKilled(true)
return false
}
validUntil := time.Unix(resp.ValidUntil, 0)
if !validUntil.After(s.now()) {
// A grant that has already ended is never relayed.
return s.refuse(cc, cur.kind, codeAuth)
}
if v := watch.verdict(resp.Host, resp.ServerTime); v.cut != "" {
// A revoke, a release or the end of the license was applied while the call was in flight: the answer is older than it. Nothing is served.
s.log.Info("authorize_overtaken", "conn", cc.id, "kind", cur.kind, "reason", v.cut)
if cur.kind != "hello" {
cc.close(v.cut)
return false
}
return s.refuse(cc, cur.kind, v.err)
}
ok := okBody{Node: s.cfg.NodeID, Host: resp.Host, Status: resp.Status, ValidUntil: resp.ValidUntil, ServerTime: resp.ServerTime, Policy: boundWindows(resp.Policy)}
// Publish and answer under the write lock, so no OPEN can be sent before the OK line.
cc.wmu.Lock()
var old *ctrl
recorded := false
if cur.kind == "hello" {
// The OK is recorded under the routing table's lock, so two HELLOs of one subject that race are recorded in the order they were routed.
old, err = s.routes.activate(cc, cur.session.Subject, resp.Host, func() {
recorded = true
s.emit(s.now(), map[string]any{"conn": cc.id, "ev": "send", "line": ok.line()})
})
if err != nil {
cc.wmu.Unlock()
s.log.Warn("host_refused", "conn", cc.id, "reason", err.Error())
return s.refuse(cc, cur.kind, codeAuth)
}
} else if cc.hostName() != resp.Host {
// The control plane now says this device owns another host: the one this connection serves is no longer its own. Fail closed.
cc.wmu.Unlock()
s.log.Warn("renew_host_changed", "conn", cc.id)
cc.close(closeRevoked)
return false
}
cc.setReady(resp, cur.session.JTI, validUntil)
s.routes.setGated(cc, resp.Status == statusGated) // every status the control plane gives, a RENEW's included, decides what the host's streams are held to
// The connection is in the routing table now, so the feed reaches it directly; what it applied before that is taken from the watch. A pause is
// put back at once (setReady wrote the answer's status over it), and the control plane is asked again.
late := watch.verdict(resp.Host, resp.ServerTime)
if late.paused {
cc.pause()
}
var werr error
if recorded {
werr = cc.sendLocked(ok.line())
} else {
werr = cc.writeLocked(ok.line(), s.now())
}
if werr == nil {
werr = s.followUp(cc, resp)
}
cc.wmu.Unlock()
if old != nil {
old.close(closeReplaced)
}
if werr != nil {
cc.close("write_error")
return false
}
if late.cut != "" {
// Applied between the check above and the connection joining the routing table: the OK is out, and the CLOSE follows it.
cc.close(late.cut)
s.routes.killStreams(resp.Host, late.cut, true)
return false
}
if late.refresh {
if late.paused {
s.routes.killStreams(resp.Host, closePaused, true)
}
s.refreshAsync(cc)
}
cc.arm(validUntil)
if cur.kind == "hello" {
s.log.Info("control_up", "conn", cc.id, "status", resp.Status, "plan", resp.Plan, "replaced_previous", old != nil)
} else {
s.log.Info("control_renewed", "conn", cc.id, "status", resp.Status, "plan", resp.Plan)
}
return true
}
// followUp writes what goes right after an OK, under the connection's write lock so that no OPEN comes between: the control plane's signed access
// statement, forwarded as it is (the device verifies it, the node cannot change a letter of it without breaking it), and DRAIN when the node is
// already draining (a connection that was authenticating when SIGTERM came).
func (s *Server) followUp(cc *ctrl, resp AuthorizeResponse) error {
if line := statementLine(resp.Access); resp.Access != "" && line != "" {
if err := cc.writeLocked(line, s.now()); err != nil {
return err
}
}
if s.draining.Load() {
return cc.writeLocked(drainLine(time.Duration(s.drainFor.Load())), s.now())
}
return nil
}
// The two ways authorize fails without having asked the control plane anything: the node's own queue was full, or its turn did not come in time.
var (
errAuthorizeFull = &ControlError{Status: 503, Code: codeControlUnavailable}
errAuthorizeWait = &ControlError{Status: 503, Code: codeControlUnavailable}
)
func isLocalFailure(err error) bool { return err == errAuthorizeFull || err == errAuthorizeWait }
// authorize asks the control plane for the verdict on one exchange. At most cfg.AuthorizeMax calls run at once; the exchanges waiting for a turn
// are bounded by the pre-authentication limit and share the call's time limit, and beyond either the device is told control_unavailable. What the
// call itself came to is recorded, so the node knows whether its control plane answers.
func (s *Server) authorize(req AuthorizeRequest) (AuthorizeResponse, error) {
if !tryAcquire(&s.authQ, int64(s.cfg.MaxUnauth)) {
s.met.authorize.inc("unavailable")
return AuthorizeResponse{}, errAuthorizeFull
}
defer s.authQ.Add(-1)
ctx, cancel := context.WithTimeout(s.ctx, s.cfg.AuthorizeTimeout)
defer cancel()
select {
case s.authSem <- struct{}{}:
defer func() { <-s.authSem }()
case <-ctx.Done():
s.met.authorize.inc("unavailable")
return AuthorizeResponse{}, errAuthorizeWait
}
resp, err := s.cfg.Control.Authorize(ctx, req)
s.health.record(s.now(), err)
switch {
case err == nil:
s.met.authorize.inc("ok")
case isRefusal(err):
s.met.authorize.inc("refused")
default:
s.met.authorize.inc("unavailable")
}
return resp, err
}
// serveAttach hands a data connection to the stream its ATTACH names. The secret is one use: a wrong one is refused and does not consume the stream,
// a used or unknown id is unknown_stream. On success there is NO reply: the next byte the node sends is the first byte of the client's stream.
func (s *Server) serveAttach(tc *tls.Conn, br *bufio.Reader, m message, line, connName string, release func(control bool)) bool {
id, secret := m.str("stream_id"), m.str("secret")
verdict := func(result string) {
s.emit(s.now(), map[string]any{"conn": connName, "ev": "attach", "line": line, "result": result})
}
s.mu.Lock()
p := s.pending[id]
switch {
case p == nil:
s.mu.Unlock()
verdict("err")
s.log.Info("attach_reject", "reason", codeUnknownStream)
writeLineThenClose(tc, errLine(codeUnknownStream))
return false
case subtle.ConstantTimeCompare([]byte(p.secret), []byte(secret)) != 1:
s.mu.Unlock()
verdict("err")
s.log.Info("attach_reject", "reason", codeAuth, "stream", id)
writeLineThenClose(tc, errLine(codeAuth))
return false
}
delete(s.pending, id) // single use
_ = tc.SetDeadline(time.Time{})
verdict("ok")
p.ch <- &bufConn{Conn: tc, r: br} // buffered(1): never blocks; the stream goroutine owns tc from here
s.mu.Unlock()
release(false)
return true
}
files/tunnel/server_test.go 1450 lines · 55.7 KB
package main
import (
"bufio"
"bytes"
"crypto/ed25519"
"crypto/rand"
"crypto/sha256"
"crypto/tls"
"encoding/hex"
"fmt"
"io"
"net"
"net/netip"
"os"
"path/filepath"
"reflect"
"regexp"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
)
// ------------------------------------------------------------------------------------------------------------------------------------
// The handshake and the label binding
// ------------------------------------------------------------------------------------------------------------------------------------
func TestHello_OKRelaysWhatTheControlPlaneDecided(t *testing.T) {
e := newTestEnv(t, nil)
// valid_until is relayed exactly as the control plane gave it, never trimmed by the node: paid 14400 s, running trial 900 s, gated 900 s
for name, tc := range map[string]struct {
validFor time.Duration
status string
plan string
}{
"paid": {secs(ctPolInt("authorization.valid_until_paid_s")), statusActive, "paid"},
"trial": {secs(ctPolInt("authorization.valid_until_trial_s")), statusActive, "trial"},
"gated": {secs(ctPolInt("authorization.valid_until_gated_s")), statusGated, "trial"},
"a trial that ends in 90 seconds": {90 * time.Second, statusActive, "trial"},
} {
d := e.newDevice()
*e.plan(d) = devicePlan{host: d.host, validFor: tc.validFor, status: tc.status, plan: tc.plan}
before := e.now().Unix()
dc := e.connectOK(d)
after := e.now().Unix()
body := dc.ok["body"].(map[string]any)
if body["host"] != d.host || body["node"] != testNodeID || body["status"] != tc.status {
t.Fatalf("%s: OK body %v", name, body)
}
until := int64(body["valid_until"].(float64))
if until < before+int64(tc.validFor/time.Second) || until > after+int64(tc.validFor/time.Second) {
t.Fatalf("%s: valid_until %d, want about %d", name, until, before+int64(tc.validFor/time.Second))
}
calls := e.cp.callLog()
last := calls[len(calls)-1]
if last.DeviceID != d.id || last.Reason != "hello" || last.Node != testNodeID {
t.Fatalf("%s: authorize call %+v", name, last)
}
// the windows are the policy's own
pol := body["policy"].(map[string]any)
if pol["dead_peer_s"] != float64(ctPolInt("timing.dead_peer_s")) || pol["renew_before_valid_until_s"] != float64(ctPolInt("timing.renew_before_valid_until_s")) {
t.Fatalf("%s: policy block %v", name, pol)
}
}
e.checkTranscript()
}
func TestLabelBinding_ASessionOfOneDeviceNeverBringsUpTheHostOfAnother(t *testing.T) {
e := newTestEnv(t, nil)
a, b := e.newDevice(), e.newDevice()
sessionA := e.session(a, nil)
attempts := map[string]string{
"host as a third field": "HELLO 2 " + sessionA + " " + b.host,
"host in the version slot": "HELLO " + b.host + " " + sessionA,
"host before the session": "HELLO 2 " + b.host + " " + sessionA,
"label as an extra field": "HELLO 2 " + sessionA + " " + b.label,
"a host command": "HOST " + b.host,
"a session for another key": "HELLO 2 " + e.sessionSignedBy(b.key, b, nil),
}
for name, line := range attempts {
dc := e.connect(a)
dc.send(line)
reply := dc.recv()
if !strings.HasPrefix(reply, "ERR ") {
t.Fatalf("%s: answered %q", name, reply)
}
dc.expectClosed(2 * time.Second)
}
// a proof that names B, or is signed by B's key, for A's session
for name, mutate := range map[string]func(h, c map[string]any){
"proof of B's subject": func(h, c map[string]any) { c["sub"] = b.id },
"proof with B's key": func(h, c map[string]any) {
h["jwk"] = map[string]any{"kty": "OKP", "crv": "Ed25519", "x": b64Encode(b.key.Public().(ed25519.PublicKey))}
},
"proof with a host claim": func(h, c map[string]any) { c["host"] = b.host; c["sub"] = b.id },
} {
dc := e.connect(a)
reply := dc.hello(func() string { return sessionA }, func(s, n string) string { return e.proof(a, s, n, mutate) })
if !strings.HasPrefix(reply, "ERR ") {
t.Fatalf("%s: answered %q", name, reply)
}
dc.expectClosed(2 * time.Second)
}
if n := len(e.cp.callLog()); n != 0 {
t.Fatalf("the control plane was asked %d times about a session that never proved itself", n)
}
if e.srv.Online(b.host) || e.srv.Online(a.host) {
t.Fatal("a host is up although no device authenticated")
}
// A, honestly: a valid session that carries extra claims naming B's host still binds only A's own host
dc := e.connect(a)
reply := dc.hello(
func() string {
return e.session(a, func(h, c map[string]any) { c["host"] = b.host; c["label"] = b.label; c["hostname"] = b.host })
},
func(s, n string) string { return e.proof(a, s, n, nil) })
if m, err := parseLine(reply, dirN2D); err != nil || m.str("verb") != "OK" || m["body"].(map[string]any)["host"] != a.host {
t.Fatalf("A's handshake: %q", reply)
}
dsA := dc.serve(nil)
if !e.srv.Online(a.host) || e.srv.Online(b.host) {
t.Fatal("the node serves the wrong host")
}
// a client of B's host while only A is connected: refused with an alert, A is never asked to open anything
if _, err := e.clientTLS(b); err == nil {
t.Fatal("a client of B's host got a TLS session")
}
time.Sleep(100 * time.Millisecond)
if dsA.openCount() != 0 {
t.Fatal("A was asked to open a stream for B's host")
}
c, err := e.clientTLS(a)
if err != nil {
t.Fatal(err)
}
if resp, err := postOver(c, a.host, []byte("hi")); err != nil || string(resp) != "echo:hi" {
t.Fatalf("A's own host: %q %v", resp, err)
}
_ = c.Close()
e.checkTranscript()
}
func TestLabelBinding_AHostIsNeverGivenToTwoDevices(t *testing.T) {
e := newTestEnv(t, nil)
a, b := e.newDevice(), e.newDevice()
// a control plane that (wrongly) answers the same host for two devices: the first holds it, the second is refused
e.plan(b).host = a.host
dcA := e.connectOK(a)
dsA := dcA.serve(nil)
dcB := e.connect(b)
reply := dcB.hello(func() string { return e.session(b, nil) }, func(s, n string) string { return e.proof(b, s, n, nil) })
if reply != "ERR auth" {
t.Fatalf("second device for the same host: %q", reply)
}
dcB.expectClosed(2 * time.Second)
// traffic for the host still reaches A
c, err := e.clientTLS(a)
if err != nil {
t.Fatal(err)
}
if resp, err := postOver(c, a.host, []byte("still-a")); err != nil || string(resp) != "echo:still-a" {
t.Fatalf("%q %v", resp, err)
}
_ = c.Close()
if dsA.openCount() != 1 {
t.Fatalf("A got %d opens", dsA.openCount())
}
e.checkTranscript()
}
func TestLabelBinding_ARenewThatChangesTheHostClosesTheConnection(t *testing.T) {
e := newTestEnv(t, nil)
d := e.newDevice()
dc := e.connectOK(d)
// the control plane now says this device owns another host (its label was retired): the old host must stop being served at once
e.plan(d).host = label20() + "." + testZone
reply := dc.renew()
_ = reply
lines := append([]string{reply}, dc.expectClosed(2*time.Second)...)
if !containsLine(lines, "CLOSE revoked") {
t.Fatalf("lines after the renew: %q", lines)
}
eventually(t, "old host unbound", 2*time.Second, func() bool { return !e.srv.Online(d.host) })
}
func containsLine(lines []string, want string) bool {
for _, l := range lines {
if l == want {
return true
}
}
return false
}
// ------------------------------------------------------------------------------------------------------------------------------------
// Proofs, sessions and what is refused
// ------------------------------------------------------------------------------------------------------------------------------------
func TestAuth_ExpiredWrongAudienceWrongKidReplayedAndWrongHTUAreRefused(t *testing.T) {
e := newTestEnv(t, nil)
d := e.newDevice()
_, rogue, _ := ed25519.GenerateKey(rand.Reader)
other := e.newDevice()
otherSession := e.session(other, nil)
type tc struct {
name string
session func() string
proof func(session, nonce string) string
want string // the ERR line
}
goodSession := func() string { return e.session(d, nil) }
goodProof := func(s, n string) string { return e.proof(d, s, n, nil) }
proofWith := func(mutate func(h, c map[string]any)) func(s, n string) string {
return func(s, n string) string { return e.proof(d, s, n, mutate) }
}
sessionWith := func(mutate func(h, c map[string]any)) func() string {
return func() string { return e.session(d, mutate) }
}
cases := []tc{
{"expired session", sessionWith(func(h, c map[string]any) { c["iat"], c["exp"] = e.now().Unix()-400, e.now().Unix()-100 }), goodProof, "ERR auth"},
{"wrong lifetime", sessionWith(func(h, c map[string]any) { c["exp"] = c["iat"].(int64) + 3000 }), goodProof, "ERR auth"},
{"audience all-relay", sessionWith(func(h, c map[string]any) { c["aud"] = "all-relay" }), goodProof, "ERR auth"},
{"audience all-control", sessionWith(func(h, c map[string]any) { c["aud"] = "all-control" }), goodProof, "ERR auth"},
{"no audience", sessionWith(func(h, c map[string]any) { delete(c, "aud") }), goodProof, "ERR auth"},
{"wrong kid", sessionWith(func(h, c map[string]any) { h["kid"] = "cp-unknown" }), goodProof, "ERR auth"},
{"signed by a forger", func() string { return e.sessionSignedBy(rogue, d, nil) }, goodProof, "ERR auth"},
{"alg none", func() string {
s := e.session(d, nil)
parts := strings.Split(s, ".")
return b64Encode([]byte(`{"alg":"none","typ":"device-session","kid":"`+testKID+`"}`)) + "." + parts[1] + "."
}, goodProof, "ERR bad_hello"}, // no signature segment: not even the shape of a token
{"alg none with a signature part", func() string {
s := e.session(d, nil)
parts := strings.Split(s, ".")
return b64Encode([]byte(`{"alg":"none","typ":"device-session","kid":"`+testKID+`"}`)) + "." + parts[1] + "." + parts[2]
}, goodProof, "ERR auth"},
{"alg HS256", sessionWith(func(h, c map[string]any) { h["alg"] = "HS256" }), goodProof, "ERR auth"},
{"wrong issuer", sessionWith(func(h, c map[string]any) { c["iss"] = "https://evil.example" }), goodProof, "ERR auth"},
{"ver 1", sessionWith(func(h, c map[string]any) { c["ver"] = 1 }), goodProof, "ERR auth"},
{"a clock far ahead", sessionWith(func(h, c map[string]any) { c["iat"], c["exp"] = e.now().Unix()+120, e.now().Unix()+420 }), goodProof, "ERR clock_skew"},
{"proof for another node's URL", goodSession, proofWith(func(h, c map[string]any) {
c["htu"] = "https://t2.nodes.all-e2e.test" + policyData.Protocol.ProofHTUPath
}), "ERR bad_proof"},
{"proof with a port in the URL", goodSession, proofWith(func(h, c map[string]any) {
c["htu"] = "https://" + testNodeHost + ":8443" + policyData.Protocol.ProofHTUPath
}), "ERR bad_proof"},
{"proof for the node's host and another path", goodSession, proofWith(func(h, c map[string]any) { c["htu"] = "https://" + testNodeHost + "/" }), "ERR bad_proof"},
{"proof with another nonce", goodSession, proofWith(func(h, c map[string]any) { c["nonce"] = "AAAAAAAAAAAAAAAAAAAAAA" }), "ERR bad_proof"},
{"proof for another session", goodSession, proofWith(func(h, c map[string]any) { c["ath"] = sha256B64([]byte(otherSession)) }), "ERR bad_proof"},
{"stale proof", goodSession, proofWith(func(h, c map[string]any) { c["iat"] = e.now().Unix() - 200 }), "ERR clock_skew"},
{"proof from the future", goodSession, proofWith(func(h, c map[string]any) { c["iat"] = e.now().Unix() + 200 }), "ERR clock_skew"},
{"proof with POST", goodSession, proofWith(func(h, c map[string]any) { c["htm"] = "POST" }), "ERR bad_proof"},
{"proof for the relay audience", goodSession, proofWith(func(h, c map[string]any) { c["aud"] = "all-relay" }), "ERR auth"},
{"proof of another subject", goodSession, proofWith(func(h, c map[string]any) { c["sub"] = other.id }), "ERR bad_proof"},
{"proof signed by another key", goodSession, func(s, n string) string { return e.proof(other, s, n, func(h, c map[string]any) { c["sub"] = d.id }) }, "ERR bad_proof"},
{"proof with a private part in the jwk", goodSession, proofWith(func(h, c map[string]any) { h["jwk"].(map[string]any)["d"] = "AAAA" }), "ERR bad_hello"},
{"proof of the wrong type", goodSession, proofWith(func(h, c map[string]any) { h["typ"] = "JWT" }), "ERR bad_proof"},
{"proof without a jti", goodSession, proofWith(func(h, c map[string]any) { delete(c, "jti") }), "ERR bad_proof"},
{"proof that is not a token", goodSession, func(s, n string) string { return "AAAA.BBBB.CCCC" }, "ERR bad_hello"},
}
for _, c := range cases {
dc := e.connect(d)
reply := dc.hello(c.session, c.proof)
if reply != c.want {
t.Errorf("%s: answered %q, want %q", c.name, reply, c.want)
}
dc.expectClosed(2 * time.Second)
if e.srv.Online(d.host) {
t.Fatalf("%s: the host is up", c.name)
}
}
if n := len(e.cp.callLog()); n != 0 {
t.Fatalf("authorize was called %d times for refused sessions and proofs", n)
}
// replay: the very proof jti of an accepted handshake is refused on another connection, with a fresh nonce
var spent string
dc := e.connect(d)
reply := dc.hello(goodSession, func(s, n string) string {
p := e.proof(d, s, n, nil)
spent = p
return p
})
if !strings.HasPrefix(reply, "OK ") {
t.Fatalf("baseline handshake: %q", reply)
}
_ = dc.c.Close()
jti := jtiOf(t, spent)
dc = e.connect(d)
reply = dc.hello(goodSession, func(s, n string) string { return e.proof(d, s, n, func(h, c map[string]any) { c["jti"] = jti }) })
if reply != "ERR bad_proof" {
t.Fatalf("replayed proof jti: %q", reply)
}
// the same proof text on a connection whose nonce differs fails on the nonce, and a session jti is NOT single use: a cached session reconnects
session := e.session(d, nil)
for i := 0; i < 2; i++ {
dc = e.connect(d)
reply = dc.hello(func() string { return session }, goodProof)
if !strings.HasPrefix(reply, "OK ") {
t.Fatalf("a cached session must reconnect (round %d): %q", i, reply)
}
_ = dc.c.Close()
}
e.checkTranscript()
}
func jtiOf(t *testing.T, token string) string {
t.Helper()
j, err := splitJWS(token, 1<<16)
if err != nil {
t.Fatal(err)
}
return j.claims["jti"].(string)
}
func TestAuth_ClockSkewWindowIsConfigurable(t *testing.T) {
ahead := func(e *testEnv, d *testDevice, by int64) string {
dc := e.connect(d)
return dc.hello(
func() string {
return e.session(d, func(h, c map[string]any) { c["iat"], c["exp"] = e.now().Unix()+by, e.now().Unix()+by+300 })
},
func(s, n string) string {
return e.proof(d, s, n, func(h, c map[string]any) { c["iat"] = e.now().Unix() + by })
})
}
e := newTestEnv(t, nil) // the policy's own window: a few seconds
d := e.newDevice()
if r := ahead(e, d, 3); !strings.HasPrefix(r, "OK ") {
t.Fatalf("a device clock 3 s fast: %q", r)
}
if r := ahead(e, d, 30); r != "ERR clock_skew" {
t.Fatalf("a device clock 30 s fast: %q", r)
}
// a device whose clock is slow still authenticates while its tokens are fresh
dc := e.connect(d)
r := dc.hello(
func() string {
return e.session(d, func(h, c map[string]any) { c["iat"], c["exp"] = e.now().Unix()-20, e.now().Unix()+280 })
},
func(s, n string) string {
return e.proof(d, s, n, func(h, c map[string]any) { c["iat"] = e.now().Unix() - 20 })
})
if !strings.HasPrefix(r, "OK ") {
t.Fatalf("a device clock 20 s slow: %q", r)
}
wide := newTestEnv(t, func(c *Config) { c.ClockSkew = 20 * time.Second })
dw := wide.newDevice()
if r := ahead(wide, dw, 15); !strings.HasPrefix(r, "OK ") {
t.Fatalf("a device clock 15 s fast inside a 20 s window: %q", r)
}
if r := ahead(wide, dw, 25); r != "ERR clock_skew" {
t.Fatalf("a device clock 25 s fast outside a 20 s window: %q", r)
}
// the window cannot be opened beyond the policy's maximum
cfg := DefaultConfig()
cfg.NodeID, cfg.NodeHost, cfg.Keys, cfg.Control = testNodeID, testNodeHost, e.keys, e.cp
cfg.CertSource = func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return nil, nil }
cfg.ClockSkew = secs(policyData.Authorization.NodeSkewMaxS) + time.Second
if _, err := NewServer(cfg); err == nil {
t.Fatal("a skew window beyond the policy's maximum was accepted")
}
}
func TestAuth_ControlPlaneRefusalsBecomeClosedWords(t *testing.T) {
e := newTestEnv(t, func(c *Config) { c.AuthorizeTimeout = 400 * time.Millisecond })
d := e.newDevice()
label := label20()
for name, tc := range map[string]struct {
plan devicePlan
want string
}{
"license ended": {devicePlan{err: &ControlError{Status: 402, Code: "license_inactive"}}, "ERR license_inactive"},
"device revoked": {devicePlan{err: &ControlError{Status: 401, Code: "device_revoked"}}, "ERR device_revoked"},
"unknown 4xx": {devicePlan{err: &ControlError{Status: 404, Code: "not_enrolled"}}, "ERR auth"},
"control plane 503": {devicePlan{err: &ControlError{Status: 503, Code: "subscription_verification_unavailable"}}, "ERR control_unavailable"},
"too many 429": {devicePlan{err: &ControlError{Status: 429, Code: "busy"}}, "ERR control_unavailable"},
"transport failure": {devicePlan{err: fmt.Errorf("connection refused")}, "ERR control_unavailable"},
"too slow": {devicePlan{delay: 3 * time.Second, host: d.host, validFor: time.Hour}, "ERR control_unavailable"},
"grant already over": {devicePlan{host: d.host, validFor: -5 * time.Second}, "ERR auth"},
"answer for another device": {devicePlan{answer: func(int) AuthorizeResponse {
return AuthorizeResponse{DeviceID: randomHex(16), Label: label, Host: label + "." + testZone, Status: "active", Plan: "paid", ValidUntil: e.now().Unix() + 600,
ServerTime: e.now().Unix(), Policy: okPolicy{20, 45, 1800, 14400, 300}}
}}, "ERR control_unavailable"},
"answer with a host that is not a device host": {devicePlan{answer: func(int) AuthorizeResponse {
return AuthorizeResponse{DeviceID: d.id, Label: "evil", Host: "evil.example.com", Status: "active", Plan: "paid", ValidUntil: e.now().Unix() + 600,
ServerTime: e.now().Unix(), Policy: okPolicy{20, 45, 1800, 14400, 300}}
}}, "ERR control_unavailable"},
} {
plan := tc.plan
if plan.host == "" && plan.err == nil && plan.answer == nil {
plan.host = d.host
}
e.cp.set(d.id, &plan)
dc := e.connect(d)
reply := dc.hello(func() string { return e.session(d, nil) }, func(s, n string) string { return e.proof(d, s, n, nil) })
if reply != tc.want {
t.Errorf("%s: %q, want %q", name, reply, tc.want)
}
dc.expectClosed(2 * time.Second)
if e.srv.Online(d.host) {
t.Fatalf("%s: the host is up", name)
}
}
e.checkTranscript()
}
// ------------------------------------------------------------------------------------------------------------------------------------
// The lifetime: valid_until, RENEW, replacement
// ------------------------------------------------------------------------------------------------------------------------------------
func TestLifetime_WithoutRenewalTheConnectionClosesAtValidUntil(t *testing.T) {
e := newTestEnv(t, nil)
d := e.newDevice()
e.plan(d).validFor = 2 * time.Second
dc := e.connectOK(d)
ds := dc.serve(nil)
until := int64(dc.ok["body"].(map[string]any)["valid_until"].(float64))
c, err := e.clientTLS(d)
if err != nil {
t.Fatal(err)
}
if resp, err := postOver(c, d.host, []byte("before")); err != nil || string(resp) != "echo:before" {
t.Fatalf("%q %v", resp, err)
}
_ = c.Close()
eventually(t, "CLOSE expired", 4*time.Second, func() bool { return containsLine(ds.otherLines(), "CLOSE expired") })
if now := e.now().Unix(); now < until {
t.Fatalf("closed at %d, before valid_until %d", now, until)
}
<-ds.done
if e.srv.Online(d.host) {
t.Fatal("the host is still up after valid_until")
}
if _, err := e.clientTLS(d); err == nil {
t.Fatal("a client got through after valid_until")
}
e.checkTranscript()
}
func TestLifetime_RenewExtendsTheConnectionAndCallsAuthorizeAgain(t *testing.T) {
e := newTestEnv(t, nil)
d := e.newDevice()
e.plan(d).validFor = 2 * time.Second
dc := e.connectOK(d)
firstUntil := int64(dc.ok["body"].(map[string]any)["valid_until"].(float64))
// the control plane's answer for the renewal is longer, and the trial turned into the paywall: gated
e.plan(d).validFor, e.plan(d).status, e.plan(d).plan = 5*time.Second, statusGated, "trial"
reply := dc.renew()
m, err := parseLine(reply, dirN2D)
if err != nil || m.str("verb") != "OK" {
t.Fatalf("RENEW answered %q", reply)
}
body := m["body"].(map[string]any)
if int64(body["valid_until"].(float64)) <= firstUntil || body["status"] != "gated" {
t.Fatalf("renewed body %v (first valid_until %d)", body, firstUntil)
}
// the renewal called authorize again, with its own session's jti and the reason renew
if e.cp.callsFor(d.id, "hello") != 1 || e.cp.callsFor(d.id, "renew") != 1 {
t.Fatalf("authorize calls: %+v", e.cp.callLog())
}
if last := e.cp.callLog()[1]; last.SessionJTI != jtiOf(t, dc.session) || last.Reason != "renew" {
t.Fatalf("the renew call %+v", last)
}
// past the FIRST valid_until the connection is still there and serves (gated still opens streams)
ds := dc.serve(nil)
time.Sleep(time.Until(time.Unix(firstUntil, 0)) + 500*time.Millisecond)
if !e.srv.Online(d.host) {
t.Fatal("the renewed connection ended at the old valid_until")
}
c, err := e.clientTLS(d)
if err != nil {
t.Fatalf("a client after the old valid_until: %v", err)
}
if resp, err := postOver(c, d.host, []byte("renewed")); err != nil || string(resp) != "echo:renewed" {
t.Fatalf("%q %v", resp, err)
}
_ = c.Close()
// and without a second renewal it ends at the NEW valid_until
eventually(t, "CLOSE expired after the renewal", 8*time.Second, func() bool { return containsLine(ds.otherLines(), "CLOSE expired") })
if now := e.now().Unix(); now < int64(body["valid_until"].(float64)) {
t.Fatalf("closed at %d, before the renewed valid_until %v", now, body["valid_until"])
}
e.checkTranscript()
}
func TestLifetime_ARefusedRenewKeepsTheConnectionAndItsOldValidUntil(t *testing.T) {
e := newTestEnv(t, nil)
d := e.newDevice()
other := e.newDevice()
e.plan(d).validFor = 3 * time.Second
dc := e.connectOK(d)
ds := dc.serve(nil)
_ = ds
// the control plane refuses the renewal
e.plan(d).err = &ControlError{Status: 402, Code: "license_inactive"}
if r := dc.renew(); r != "ERR license_inactive" {
t.Fatalf("refused RENEW answered %q", r)
}
e.plan(d).err = nil
// a RENEW with the session of another device
dc.send("RENEW " + e.session(other, nil))
if r := dc.recv(); r != "ERR auth" {
t.Fatalf("RENEW of another subject: %q", r)
}
// a RENEW whose proof is bad
dc.send("RENEW " + e.session(d, nil))
if line := dc.recv(); !strings.HasPrefix(line, "CHALLENGE ") {
t.Fatalf("%q", line)
}
dc.send("PROOF AAAA.BBBB.CCCC")
if r := dc.recv(); !strings.HasPrefix(r, "ERR ") {
t.Fatalf("%q", r)
}
// a session that is expired
dc.send("RENEW " + e.session(d, func(h, c map[string]any) { c["iat"], c["exp"] = e.now().Unix()-400, e.now().Unix()-100 }))
if r := dc.recv(); r != "ERR auth" {
t.Fatalf("%q", r)
}
// the connection is alive and answers PING: nothing above extended or ended it
dc.send("PING")
deadline := time.Now().Add(2 * time.Second)
for !containsLine(ds.otherLines(), "PONG") && time.Now().Before(deadline) {
time.Sleep(20 * time.Millisecond)
}
if !e.srv.Online(d.host) {
t.Fatal("a refused RENEW ended the connection")
}
eventually(t, "CLOSE expired at the OLD valid_until", 5*time.Second, func() bool { return containsLine(ds.otherLines(), "CLOSE expired") })
e.checkTranscript()
}
func TestLifetime_ARenewThatNeverGetsItsProofIsDroppedNotTheConnection(t *testing.T) {
e := newTestEnv(t, func(c *Config) { c.FirstLineTimeout = 400 * time.Millisecond })
d := e.newDevice()
dc := e.connectOK(d)
dc.send("RENEW " + e.session(d, nil))
if line := dc.recv(); !strings.HasPrefix(line, "CHALLENGE ") {
t.Fatalf("%q", line)
}
time.Sleep(900 * time.Millisecond)
// the abandoned exchange's PROOF finds no nonce; a fresh RENEW works
dc.send("PING")
if line := dc.recv(); line != "PONG" {
t.Fatalf("%q", line)
}
if r := dc.renew(); !strings.HasPrefix(r, "OK ") {
t.Fatalf("a new RENEW after the abandoned one: %q", r)
}
}
func TestLifetime_ASecondHelloOfTheSameSubjectReplacesTheFirst(t *testing.T) {
e := newTestEnv(t, nil)
d := e.newDevice()
first := e.connectOK(d)
dsFirst := first.serve(nil)
// a stream that is open on the first connection when it is replaced keeps running
c, err := e.clientTLS(d)
if err != nil {
t.Fatal(err)
}
if resp, err := postOver(c, d.host, []byte("one")); err != nil || string(resp) != "echo:one" {
t.Fatalf("%q %v", resp, err)
}
if dsFirst.openCount() != 1 {
t.Fatalf("first opens %d", dsFirst.openCount())
}
second := e.connectOK(d)
dsSecond := second.serve(nil)
eventually(t, "CLOSE replaced on the first connection", 3*time.Second, func() bool { return containsLine(dsFirst.otherLines(), "CLOSE replaced") })
<-dsFirst.done
if resp, err := postOver(c, d.host, []byte("two")); err != nil || string(resp) != "echo:two" {
t.Fatalf("the open stream after the replacement: %q %v", resp, err)
}
_ = c.Close()
c2, err := e.clientTLS(d)
if err != nil {
t.Fatal(err)
}
if resp, err := postOver(c2, d.host, []byte("three")); err != nil || string(resp) != "echo:three" {
t.Fatalf("%q %v", resp, err)
}
_ = c2.Close()
if dsSecond.openCount() != 1 || dsFirst.openCount() != 1 {
t.Fatalf("opens: first %d second %d", dsFirst.openCount(), dsSecond.openCount())
}
e.checkTranscript()
}
func TestLifetime_ManyConcurrentHellosOfOneSubjectLeaveExactlyOneConnection(t *testing.T) {
e := newTestEnv(t, nil)
d := e.newDevice()
const n = 24
var wg sync.WaitGroup
conns := make([]*devConn, n)
var failures atomic.Int32
for i := 0; i < n; i++ {
c, err := e.dialNode()
if err != nil {
t.Fatal(err)
}
conns[i] = &devConn{t: t, e: e, d: d, c: c, br: bufio.NewReader(c)}
t.Cleanup(func() { _ = c.Close() })
}
for i := 0; i < n; i++ {
wg.Add(1)
go func(dc *devConn) {
defer wg.Done()
dc.session = e.session(d, nil)
fmt.Fprintf(dc.c, "HELLO 2 %s\n", dc.session)
line, err := dc.br.ReadString('\n')
if err != nil || !strings.HasPrefix(line, "CHALLENGE ") {
t.Logf("challenge: %q %v", line, err)
failures.Add(1)
return
}
fmt.Fprintf(dc.c, "PROOF %s\n", e.proof(d, dc.session, strings.TrimSpace(strings.TrimPrefix(line, "CHALLENGE ")), nil))
if line, err = dc.br.ReadString('\n'); err != nil || !strings.HasPrefix(line, "OK ") {
t.Logf("ok: %q %v", line, err)
failures.Add(1)
}
}(conns[i])
}
wg.Wait()
if failures.Load() != 0 {
t.Fatalf("%d handshakes failed", failures.Load())
}
// exactly one connection is left open; every other one was told it was replaced
open := 0
for _, dc := range conns {
_ = dc.c.SetReadDeadline(time.Now().Add(1500 * time.Millisecond))
line, err := dc.br.ReadString('\n')
if err == nil && line == "CLOSE replaced\n" {
continue
}
open++
}
if open != 1 {
t.Fatalf("%d connections left open", open)
}
if !e.srv.Online(d.host) {
t.Fatal("the host is not served by the survivor")
}
e.srv.routes.mu.Lock()
if len(e.srv.routes.bySub) != 1 || len(e.srv.routes.byHost) != 1 {
t.Errorf("the table holds %d subjects and %d hosts", len(e.srv.routes.bySub), len(e.srv.routes.byHost))
}
e.srv.routes.mu.Unlock()
e.checkTranscript()
}
func TestLifetime_APausedDeviceKeepsItsConnectionButOpensNothing(t *testing.T) {
e := newTestEnv(t, nil)
d := e.newDevice()
e.plan(d).status = statusPaused
dc := e.connectOK(d)
ds := dc.serve(nil)
if dc.ok["body"].(map[string]any)["status"] != "paused" {
t.Fatalf("%v", dc.ok)
}
if _, err := e.clientTLS(d); err == nil {
t.Fatal("a client got a stream of a paused device")
}
time.Sleep(100 * time.Millisecond)
if ds.openCount() != 0 {
t.Fatal("a paused device was asked to open a stream")
}
// resumed: the renewal says active and streams flow again, on the same connection
e.plan(d).status = statusActive
r := dc.renew()
if !strings.HasPrefix(r, "OK ") {
t.Fatalf("%q", r)
}
c, err := e.clientTLS(d)
if err != nil {
t.Fatal(err)
}
_ = c.Close()
e.checkTranscript()
}
func TestLifetime_ADeadPeerIsDroppedAndAPingKeepsItAlive(t *testing.T) {
e := newTestEnv(t, func(c *Config) { c.DeadPeer = 600 * time.Millisecond })
d := e.newDevice()
dc := e.connectOK(d)
for i := 0; i < 4; i++ {
time.Sleep(250 * time.Millisecond)
dc.send("PING")
if line := dc.recv(); line != "PONG" {
t.Fatalf("%q", line)
}
}
if !e.srv.Online(d.host) {
t.Fatal("a pinging device was dropped")
}
lines := dc.expectClosed(3 * time.Second)
if !containsLine(lines, "CLOSE idle") {
t.Fatalf("lines before the close: %q", lines)
}
eventually(t, "host unbound", time.Second, func() bool { return !e.srv.Online(d.host) })
}
// ------------------------------------------------------------------------------------------------------------------------------------
// ATTACH
// ------------------------------------------------------------------------------------------------------------------------------------
// openOne makes a client arrive and returns its OPEN line, parsed; the client's goroutine ends when done is closed or the handshake gives up.
func openOne(t *testing.T, e *testEnv, d *testDevice, dc *devConn) (message, chan<- struct{}) {
t.Helper()
stop := make(chan struct{})
go func() {
c, err := e.clientTLS(d)
if err == nil {
<-stop
_ = c.Close()
}
}()
line := dc.recv()
m, err := parseLine(line, dirN2D)
if err != nil || m.str("verb") != "OPEN" {
t.Fatalf("expected an OPEN, got %q", line)
}
return m, stop
}
func attachLine(id, secret string) string { return "ATTACH " + id + " " + secret }
func TestAttach_AWrongOrReusedSecretFailsAndTheRightOneWorksOnce(t *testing.T) {
e := newTestEnv(t, nil)
d := e.newDevice()
dc := e.connectOK(d)
open, stop := openOne(t, e, d, dc)
defer close(stop)
id, secret := open.str("stream_id"), open.str("secret")
rawAttach := func(line string) (reply string, closed bool) {
c, err := e.dialNode()
if err != nil {
t.Fatal(err)
}
defer c.Close()
fmt.Fprintf(c, "%s\n", line)
_ = c.SetReadDeadline(time.Now().Add(3 * time.Second))
br := bufio.NewReader(c)
b, _ := br.ReadString('\n')
_, err = br.ReadByte() // the node closes after an ERR
return strings.TrimSpace(b), err != nil
}
wrong := secret[:len(secret)-1] + map[bool]string{true: "B", false: "A"}[secret[len(secret)-1] == 'A']
if r, closed := rawAttach(attachLine(id, wrong)); r != "ERR auth" || !closed {
t.Fatalf("wrong secret: %q closed=%v", r, closed)
}
if r, _ := rawAttach(attachLine(id, strings.Repeat("A", 43))); r != "ERR auth" {
t.Fatalf("zero secret: %q", r)
}
// a wrong secret does not consume the stream: the right one still works
c, err := e.dialNode()
if err != nil {
t.Fatal(err)
}
defer c.Close()
fmt.Fprintf(c, "%s\n", attachLine(id, secret))
// success has no reply: the next byte is the client's ClientHello (a TLS handshake record)
_ = c.SetReadDeadline(time.Now().Add(3 * time.Second))
first := make([]byte, 1)
if _, err := io.ReadFull(c, first); err != nil || first[0] != 0x16 {
t.Fatalf("after a good ATTACH the stream's first byte is % x, err %v", first, err)
}
// used: the same id and secret again is an unknown stream
if r, closed := rawAttach(attachLine(id, secret)); r != "ERR unknown_stream" || !closed {
t.Fatalf("reused secret: %q", r)
}
if r, _ := rawAttach(attachLine(randomHex(16), secret)); r != "ERR unknown_stream" {
t.Fatalf("unknown id: %q", r)
}
for _, bad := range []string{"ATTACH", "ATTACH " + id, "ATTACH " + strings.ToUpper(id) + " " + secret, "ATTACH " + id + " short", "ATTACH " + id + " " + secret + " extra"} {
if r, _ := rawAttach(bad); r != "ERR bad_attach" && r != "ERR bad_command" {
t.Errorf("%q answered %q", bad, r)
}
}
e.checkTranscript()
}
func TestAttach_AfterTheTimeoutTheStreamIsGone(t *testing.T) {
e := newTestEnv(t, func(c *Config) { c.AttachTimeout = 300 * time.Millisecond })
d := e.newDevice()
dc := e.connectOK(d)
open, stop := openOne(t, e, d, dc)
defer close(stop)
time.Sleep(700 * time.Millisecond)
c, err := e.dialNode()
if err != nil {
t.Fatal(err)
}
defer c.Close()
fmt.Fprintf(c, "%s\n", attachLine(open.str("stream_id"), open.str("secret")))
_ = c.SetReadDeadline(time.Now().Add(3 * time.Second))
if b, _ := bufio.NewReader(c).ReadString('\n'); strings.TrimSpace(b) != "ERR unknown_stream" {
t.Fatalf("attach after the timeout: %q", b)
}
e.waitIdle()
// (no transcript check here: the checker judges a late ATTACH by the policy's attach window, this node runs a shorter one)
}
func TestAttach_EveryStreamGetsItsOwnIdAndSecret(t *testing.T) {
e := newTestEnv(t, nil)
d := e.newDevice()
dc := e.connectOK(d)
ds := dc.serve(func(c net.Conn) { time.Sleep(50 * time.Millisecond) })
for i := 0; i < 40; i++ {
c, err := tls.DialWithDialer(&net.Dialer{Timeout: time.Second}, "tcp", e.httpsAddr, &tls.Config{ServerName: d.host, InsecureSkipVerify: true})
if c != nil {
_ = c.Close()
}
_ = err
}
eventually(t, "40 opens", 5*time.Second, func() bool { return ds.openCount() >= 40 })
ids, secrets := map[string]bool{}, map[string]bool{}
for _, m := range ds.openList() {
ids[m.str("stream_id")], secrets[m.str("secret")] = true, true
if !reStreamID.MatchString(m.str("stream_id")) || !reSecret.MatchString(m.str("secret")) {
t.Fatalf("OPEN %v", m)
}
}
if len(ids) != 40 || len(secrets) != 40 {
t.Fatalf("%d distinct ids, %d distinct secrets for 40 streams", len(ids), len(secrets))
}
}
// ------------------------------------------------------------------------------------------------------------------------------------
// OPEN only after a complete, well-formed ClientHello
// ------------------------------------------------------------------------------------------------------------------------------------
// helloOf builds a ClientHello for host with a padding extension of pad bytes, as records of at most recordSize payload bytes.
func helloOf(host string, pad, recordSize int) []byte {
ext := sniExt(host)
if pad > 0 {
ext = append(append([]byte{0, 21, byte(pad >> 8), byte(pad)}, make([]byte, pad)...), ext...)
}
var body []byte
body = append(body, 3, 3)
body = append(body, make([]byte, 32)...)
body = append(body, 0)
body = append(body, 0, 2, 0x13, 0x01)
body = append(body, 1, 0)
body = append(body, byte(len(ext)>>8), byte(len(ext)))
body = append(body, ext...)
hs := append([]byte{1, byte(len(body) >> 16), byte(len(body) >> 8), byte(len(body))}, body...)
var out []byte
for len(hs) > 0 {
k := min(recordSize, len(hs))
out = append(out, 22, 3, 1, byte(k>>8), byte(k))
out = append(out, hs[:k]...)
hs = hs[k:]
}
return out
}
// sendRaw opens a connection, writes b in pieces of size (0 = at once) and returns it.
func sendRaw(t *testing.T, e *testEnv, b []byte, size int) net.Conn {
t.Helper()
c, err := net.DialTimeout("tcp", e.httpsAddr, 3*time.Second)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = c.Close() })
var w net.Conn = c
if size > 0 {
w = chunkConn{Conn: c, size: size, delay: time.Millisecond}
}
_, _ = w.Write(b)
return c
}
func TestClientHello_NothingThatIsNotACompleteWellFormedHelloProducesAnOpen(t *testing.T) {
e := newTestEnv(t, func(c *Config) { c.HelloTimeout = 400 * time.Millisecond })
d := e.newDevice()
dc := e.connectOK(d)
ds := dc.serve(nil)
ds.noAttach = true
full := helloOf(d.host, 0, 16000)
t.Run("truncated at every interesting length", func(t *testing.T) {
for _, n := range []int{1, 2, 3, 4, 5, 6, 9, 20, 43, 44, len(full) / 2, len(full) - 1} {
c := sendRaw(t, e, full[:n], 0)
_ = c.Close() // the client goes away mid-hello
}
})
t.Run("stalls half way", func(t *testing.T) {
c := sendRaw(t, e, full[:len(full)/2], 0)
_ = c.SetReadDeadline(time.Now().Add(3 * time.Second))
start := time.Now()
if n, err := c.Read(make([]byte, 1)); n != 0 || err == nil {
t.Fatalf("n=%d err=%v", n, err)
}
if time.Since(start) > 2*time.Second {
t.Fatal("the hello deadline is not enforced")
}
})
t.Run("a hello whose record length lies", func(t *testing.T) {
lie := append([]byte(nil), full...)
lie[3], lie[4] = 0xff, 0xff
sendRaw(t, e, lie, 0)
})
t.Run("a handshake that is not a ClientHello", func(t *testing.T) {
notHello := append([]byte(nil), full...)
notHello[5] = 2
sendRaw(t, e, notHello, 0)
})
t.Run("not TLS at all", func(t *testing.T) {
sendRaw(t, e, []byte("GET / HTTP/1.1\r\nHost: "+d.host+"\r\n\r\n"), 0)
})
t.Run("no SNI", func(t *testing.T) {
sendRaw(t, e, realHello(t, ""), 0)
})
t.Run("an address as the name", func(t *testing.T) {
sendRaw(t, e, buildHello(sniExt("192.0.2.3"), true), 0)
})
t.Run("another host", func(t *testing.T) {
sendRaw(t, e, helloOf(label20()+"."+testZone, 0, 16000), 0)
})
t.Run("a name with an underscore, a double dot, a space", func(t *testing.T) {
for _, name := range []string{"a_b." + testZone, "a..b." + testZone, "a b." + testZone, d.label + "." + testZone + ".."} {
sendRaw(t, e, helloOf(name, 0, 16000), 0)
}
})
time.Sleep(900 * time.Millisecond)
if n := ds.openCount(); n != 0 {
t.Fatalf("%d OPEN lines for hellos that were not complete and well formed", n)
}
// teeth: the same bytes, whole, do produce one
sendRaw(t, e, full, 0)
eventually(t, "the OPEN of the complete hello", 3*time.Second, func() bool { return ds.openCount() == 1 })
e.checkTranscript()
}
func TestClientHello_FragmentedHugeCasedAndDottedHellosAreServed(t *testing.T) {
e := newTestEnv(t, nil)
d := e.newDevice()
dc := e.connectOK(d)
ds := dc.serve(nil)
ds.noAttach = true
variants := map[string][]byte{
"split in 100 byte records": helloOf(d.host, 0, 100),
"in 7 byte records": helloOf(d.host, 0, 7),
"64 KB": helloOf(d.host, 65000, 16384),
"upper case": helloOf(strings.ToUpper(d.host), 0, 16000),
"mixed case": helloOf(strings.ToUpper(d.label[:5])+d.label[5:]+"."+strings.ToUpper(testZone[:3])+testZone[3:], 0, 16000),
"trailing dot": helloOf(d.host+".", 0, 16000),
"a real client's hello": realHello(t, d.host),
"a client hello offering X25519MLKEM768": captureClientHello(t, &tls.Config{ServerName: d.host, InsecureSkipVerify: true, CurvePreferences: []tls.CurveID{tls.X25519MLKEM768, tls.X25519}}),
}
want := 0
for name, hello := range variants {
_ = name
want++
sendRaw(t, e, hello, 0)
}
eventually(t, "an OPEN per variant", 5*time.Second, func() bool { return ds.openCount() == want })
// arriving one byte at a time (over a pipe so the pieces really are separate reads)
sendRaw(t, e, helloOf(d.host, 0, 16000), 1)
eventually(t, "the byte-by-byte hello", 10*time.Second, func() bool { return ds.openCount() == want+1 })
e.checkTranscript()
}
func TestClientHello_ThePeerAddressAndTheClassTravelInTheOpen(t *testing.T) {
cases := map[string]struct {
classify func(netip.Addr, string) string
want string
}{
"no classifier": {nil, "anon"},
"openai": {func(netip.Addr, string) string { return "openai" }, "openai"},
"hint": {func(netip.Addr, string) string { return "hint" }, "hint"},
"a class that is not": {func(netip.Addr, string) string { return "vip" }, "anon"},
}
for name, tc := range cases {
var seenPeer atomic.Value
e := newTestEnv(t, func(c *Config) {
if tc.classify != nil {
c.Classify = func(peer netip.Addr, host string) string {
seenPeer.Store(peer.String() + " " + host)
return tc.classify(peer, host)
}
}
})
d := e.newDevice()
dc := e.connectOK(d)
ds := dc.serve(nil)
ds.noAttach = true
sendRaw(t, e, helloOf(d.host, 0, 16000), 0)
eventually(t, name+": the OPEN", 3*time.Second, func() bool { return ds.openCount() == 1 })
open := ds.openList()[0]
if open.str("peer_ip") != "127.0.0.1" || open.str("class") != tc.want {
t.Errorf("%s: OPEN %v", name, open)
}
if tc.classify != nil && seenPeer.Load() != "127.0.0.1 "+d.host {
t.Errorf("%s: the classifier was asked about %v", name, seenPeer.Load())
}
e.checkTranscript()
}
}
// ------------------------------------------------------------------------------------------------------------------------------------
// Port 80 and the listeners
// ------------------------------------------------------------------------------------------------------------------------------------
func acmeGet(t *testing.T, e *testEnv, method, path, host string) string {
t.Helper()
return rawExchange(t, e.httpAddr, method+" "+path+" HTTP/1.1\r\nHost: "+host+"\r\nConnection: close\r\n\r\n")
}
func TestPort80_ServesOnlyTheNodesOwnACMEWebroot(t *testing.T) {
webroot := t.TempDir()
dir := filepath.Join(webroot, ".well-known", "acme-challenge")
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
must := func(err error) {
t.Helper()
if err != nil {
t.Fatal(err)
}
}
must(os.WriteFile(filepath.Join(dir, "tok_OK-1"), []byte("tok_OK-1.thumbprint"), 0o644))
must(os.WriteFile(filepath.Join(dir, "huge"), bytes.Repeat([]byte("x"), 9000), 0o644))
outside := filepath.Join(t.TempDir(), "secret")
must(os.WriteFile(outside, []byte("not for the internet"), 0o644))
must(os.Symlink(outside, filepath.Join(dir, "linked")))
must(os.WriteFile(filepath.Join(webroot, "private.txt"), []byte("private"), 0o644))
e := newTestEnv(t, func(c *Config) { c.ACMEWebroot = webroot })
d := e.newDevice()
dc := e.connectOK(d)
ds := dc.serve(nil)
ds.noAttach = true
if out := acmeGet(t, e, "GET", "/.well-known/acme-challenge/tok_OK-1", testNodeHost); !strings.HasPrefix(out, "HTTP/1.1 200 OK") || !strings.HasSuffix(out, "tok_OK-1.thumbprint") {
t.Fatalf("the node's own challenge: %q", out)
}
if out := acmeGet(t, e, "GET", "/.well-known/acme-challenge/tok_OK-1", strings.ToUpper(testNodeHost)+":80"); !strings.HasPrefix(out, "HTTP/1.1 200 OK") {
t.Fatalf("the node's own name, upper case with a port: %q", out)
}
if out := acmeGet(t, e, "HEAD", "/.well-known/acme-challenge/tok_OK-1", testNodeHost); !strings.HasPrefix(out, "HTTP/1.1 200 OK") || strings.Contains(out, "thumbprint") {
t.Fatalf("HEAD: %q", out)
}
for name, req := range map[string][2]string{
"a device's host": {"/.well-known/acme-challenge/tok_OK-1", d.host},
"another host": {"/.well-known/acme-challenge/tok_OK-1", "example.com"},
"no such token": {"/.well-known/acme-challenge/nope", testNodeHost},
"the webroot itself": {"/", testNodeHost},
"a file outside the path": {"/private.txt", testNodeHost},
"traversal": {"/.well-known/acme-challenge/../../private.txt", testNodeHost},
"encoded traversal": {"/.well-known/acme-challenge/%2e%2e/private.txt", testNodeHost},
"a query": {"/.well-known/acme-challenge/tok_OK-1?x=1", testNodeHost},
"a file that is too large": {"/.well-known/acme-challenge/huge", testNodeHost},
"a symlink": {"/.well-known/acme-challenge/linked", testNodeHost},
"a metrics path": {"/metrics", testNodeHost},
"healthz": {"/healthz", testNodeHost},
} {
if out := acmeGet(t, e, "GET", req[0], req[1]); !strings.HasPrefix(out, "HTTP/1.1 404 ") || strings.Contains(out, "thumbprint") || strings.Contains(out, "private") || strings.Contains(out, "internet") {
t.Errorf("%s: %q", name, out)
}
}
if out := acmeGet(t, e, "POST", "/.well-known/acme-challenge/tok_OK-1", testNodeHost); !strings.HasPrefix(out, "HTTP/1.1 404 ") {
t.Errorf("POST: %q", out)
}
if out := rawExchange(t, e.httpAddr, "GET /.well-known/acme-challenge/tok_OK-1 HTTP/1.0\r\n\r\n"); out != "" && !strings.HasPrefix(out, "HTTP/1.1 404 ") {
t.Errorf("no Host: %q", out)
}
time.Sleep(100 * time.Millisecond)
if ds.openCount() != 0 {
t.Fatal("port 80 asked a device to open a stream: it must never reach one")
}
}
func TestPort80_WithoutAWebrootEverythingIs404(t *testing.T) {
e := newTestEnv(t, nil)
if out := acmeGet(t, e, "GET", "/.well-known/acme-challenge/anything", testNodeHost); !strings.HasPrefix(out, "HTTP/1.1 404 ") {
t.Fatalf("%q", out)
}
}
func TestListeners_ThereIsNoPort8443AndNoSelfSignedPin(t *testing.T) {
e := newTestEnv(t, nil)
if e.srv.httpsLn == nil || e.srv.httpLn == nil {
t.Fatal("the two listeners are not up")
}
// the node holds exactly two listeners (the field set of the server says it: no control listener exists)
typ := reflect.TypeOf(e.srv).Elem()
for i := 0; i < typ.NumField(); i++ {
if name := typ.Field(i).Name; strings.Contains(strings.ToLower(name), "ctrlln") || strings.Contains(strings.ToLower(name), "controlln") {
t.Fatalf("the server has a control listener field %s", name)
}
}
cfgType := reflect.TypeOf(Config{})
for i := 0; i < cfgType.NumField(); i++ {
if name := cfgType.Field(i).Name; strings.Contains(name, "ControlAddr") || strings.Contains(name, "StateDir") || strings.Contains(name, "Devices") {
t.Fatalf("Config still has %s", name)
}
}
// nothing in the sources speaks of :8443, a device token or a pinned fingerprint any more
files, _ := filepath.Glob("*.go")
forbidden := regexp.MustCompile(`8443|ControlAddr|control_cert_sha256|server\.fingerprint|loadOrCreateServerCert|checkToken`)
for _, f := range files {
if strings.HasSuffix(f, "_test.go") {
continue
}
b, _ := os.ReadFile(f)
if loc := forbidden.FindIndex(b); loc != nil {
t.Errorf("%s still mentions %q", f, b[loc[0]:loc[1]])
}
}
}
// ------------------------------------------------------------------------------------------------------------------------------------
// The device's connection to the node's own name
// ------------------------------------------------------------------------------------------------------------------------------------
func TestOwnName_OnlyAllTunnel2OverTLS13AndOnlyHelloOrAttach(t *testing.T) {
e := newTestEnv(t, nil)
reply := func(line string) string {
c, err := e.dialNode()
if err != nil {
t.Fatal(err)
}
defer c.Close()
_, _ = io.WriteString(c, line+"\n")
_ = c.SetReadDeadline(time.Now().Add(3 * time.Second))
b, _ := bufio.NewReader(c).ReadString('\n')
return strings.TrimSpace(b)
}
for line, want := range map[string]string{
"hello 2 AAAA.BBBB.CCCC": "ERR bad_command",
"PING": "ERR bad_command",
"PROOF not-a-token": "ERR bad_command", // a malformed PROOF/RENEW as the FIRST line is not HELLO or ATTACH (contract section 2)
"PROOF AAAA.BBBB.CCCC": "ERR bad_command",
"PROOF": "ERR bad_command",
"RENEW not-a-token": "ERR bad_command",
"RENEW AAAA.BBBB.CCCC": "ERR bad_command",
"OPEN " + randomHex(16): "ERR bad_command",
"HELLO 3 AAAA.BBBB.CCCC": "ERR unsupported_version",
"HELLO 1 AAAA.BBBB.CCCC": "ERR unsupported_version",
"HELLO two AAAA.BBBB.CCCC": "ERR bad_command",
"HELLO 2": "ERR bad_command",
"HELLO 2 not-a-token": "ERR bad_hello",
"HELLO 2 AAAA.BBBB.CCCC extra": "ERR bad_command",
"HELLO 2 AAAA.BBBB.CCCC": "ERR bad_command",
"HELLO 2 AAAA.BBBB.CCCC\r": "ERR bad_command",
"GET / HTTP/1.1": "ERR bad_command",
"ATTACH": "ERR bad_command",
"ATTACH short short": "ERR bad_attach",
strings.Repeat("A", 9000): "ERR line_too_long",
"HELLO 2 " + strings.Repeat("A", 5000) + ".B.C": "ERR bad_hello",
} {
if got := reply(line); got != want {
t.Errorf("%.40q: %q, want %q", line, got, want)
}
}
dial := func(cfg *tls.Config) error {
cfg.ServerName, cfg.RootCAs = testNodeHost, e.nodePool
c, err := tls.DialWithDialer(&net.Dialer{Timeout: 3 * time.Second}, "tcp", e.httpsAddr, cfg)
if err == nil {
_ = c.SetReadDeadline(time.Now().Add(2 * time.Second))
_, err = c.Read(make([]byte, 1)) // the node closes: it does not talk to a client that did not offer the protocol
_ = c.Close()
}
return err
}
if err := dial(&tls.Config{NextProtos: []string{"h2"}, MinVersion: tls.VersionTLS13}); err == nil {
t.Error("a client offering only h2 got a session")
}
if err := dial(&tls.Config{NextProtos: []string{"http/1.1"}, MinVersion: tls.VersionTLS13}); err == nil {
t.Error("a client offering only http/1.1 stayed")
}
if err := dial(&tls.Config{MinVersion: tls.VersionTLS13}); err == nil {
t.Error("a client offering no ALPN stayed")
}
if err := dial(&tls.Config{NextProtos: []string{policyData.Protocol.ALPN}, MaxVersion: tls.VersionTLS12}); err == nil {
t.Error("a TLS 1.2 client got a session")
}
// ... and the one that speaks the protocol is not hurt by the refusals
d := e.newDevice()
_ = e.connectOK(d)
}
func TestOwnName_UnauthenticatedConnectionsAreCapped(t *testing.T) {
e := newTestEnv(t, func(c *Config) {
c.MaxUnauth = 4
c.FirstLineTimeout = 700 * time.Millisecond
c.HelloTimeout = 700 * time.Millisecond
})
var held []net.Conn
for i := 0; i < 4; i++ {
c, err := net.Dial("tcp", e.httpsAddr)
if err != nil {
t.Fatal(err)
}
held = append(held, c)
defer c.Close()
}
time.Sleep(150 * time.Millisecond)
// the fifth is turned away without a byte
c, err := net.Dial("tcp", e.httpsAddr)
if err != nil {
t.Fatal(err)
}
defer c.Close()
_ = c.SetReadDeadline(time.Now().Add(2 * time.Second))
if n, err := c.Read(make([]byte, 1)); n != 0 || err == nil {
t.Fatalf("n=%d err=%v", n, err)
}
// once the held ones time out, the node serves again
time.Sleep(900 * time.Millisecond)
d := e.newDevice()
_ = e.connectOK(d)
_ = held
}
// A control plane that is slow to answer must not starve what the node does without it: a device that proved itself is no longer an
// unauthenticated connection, so the clients (and the ATTACHes) of devices that are already connected go on.
func TestOwnName_SlowAuthorizeDoesNotStarveClientsOfConnectedDevices(t *testing.T) {
e := newTestEnv(t, func(c *Config) { c.MaxUnauth = 4 })
d := e.newDevice()
dc := e.connectOK(d)
_ = dc.serve(nil)
var pending []<-chan string
for i := 0; i < 4; i++ {
slow := e.newDevice()
e.plan(slow).delay = 2 * time.Second
pending = append(pending, e.helloAsync(slow))
}
eventually(t, "the slow authorize calls are in flight", 3*time.Second, func() bool { return len(e.cp.callLog()) >= 5 })
// a client of the connected device is served at once (its stream's ATTACH goes through the same gate)
c, err := e.clientTLS(d)
if err != nil {
t.Fatalf("a client of a connected device was turned away while authorize was slow: %v", err)
}
defer c.Close()
if resp, err := postOver(c, d.host, []byte("hello")); err != nil || !bytes.Contains(resp, []byte("hello")) {
t.Fatalf("resp=%q err=%v", resp, err)
}
for _, p := range pending {
if got := <-p; !strings.HasPrefix(got, "OK ") {
t.Fatalf("a slow device ended with %q", got)
}
}
e.checkTranscript()
}
// At most AuthorizeMax calls to the control plane run at once; the ones that wait for a turn are bounded too, and beyond that the device is told
// control_unavailable.
func TestAuthorize_ConcurrencyAndTheWaitingAreBounded(t *testing.T) {
e := newTestEnv(t, func(c *Config) { c.AuthorizeMax = 2 })
var pending []<-chan string
for i := 0; i < 6; i++ {
d := e.newDevice()
e.plan(d).delay = 300 * time.Millisecond
pending = append(pending, e.helloAsync(d))
}
for _, p := range pending {
if got := <-p; !strings.HasPrefix(got, "OK ") {
t.Fatalf("%q", got)
}
}
e.cp.mu.Lock()
most := e.cp.maxInflight
e.cp.mu.Unlock()
if most != 2 {
t.Fatalf("%d authorize calls at once, want exactly the limit of 2", most)
}
full := newTestEnv(t, func(c *Config) { c.AuthorizeMax, c.MaxUnauth = 1, 2 })
pending = nil
for i := 0; i < 3; i++ {
d := full.newDevice()
full.plan(d).delay = time.Second
pending = append(pending, full.helloAsync(d))
}
ok, refused := 0, 0
for _, p := range pending {
switch got := <-p; {
case strings.HasPrefix(got, "OK "):
ok++
case got == "ERR control_unavailable":
refused++
default:
t.Fatalf("%q", got)
}
}
if ok != 2 || refused != 1 {
t.Fatalf("%d OK and %d refused, want 2 and 1", ok, refused)
}
}
// What the node records of two connections of one subject is in the order it routed them: the OK is recorded under the routing table's lock.
func TestRoutes_ActivationIsRecordedUnderTheTableLock(t *testing.T) {
rt := newRouteTable()
a := &ctrl{}
underLock := false
if _, err := rt.activate(a, "sub", "h.example", func() {
if rt.mu.TryLock() {
rt.mu.Unlock()
return
}
underLock = true
}); err != nil || !underLock {
t.Fatalf("err=%v, recorded under the lock: %v", err, underLock)
}
if _, err := rt.activate(&ctrl{}, "other", "h.example", func() { t.Error("recorded a refused activation") }); err != errHostTaken {
t.Fatalf("err=%v", err)
}
}
func TestHint_AddressesReachTheHookAndABadHintClosesTheConnection(t *testing.T) {
type hint struct {
host string
addrs []netip.Addr
}
got := make(chan hint, 4)
e := newTestEnv(t, func(c *Config) { c.OnHint = func(host string, addrs []netip.Addr) { got <- hint{host, addrs} } })
d := e.newDevice()
dc := e.connectOK(d)
dc.send("HINT 203.0.113.9 2001:db8::7")
select {
case h := <-got:
if h.host != d.host || len(h.addrs) != 2 || h.addrs[0].String() != "203.0.113.9" || h.addrs[1].String() != "2001:db8::7" {
t.Fatalf("%+v", h)
}
case <-time.After(3 * time.Second):
t.Fatal("no hint")
}
dc.send("HINT 203.0.113.9 not-an-ip")
if r := dc.recv(); r != "ERR bad_command" {
t.Fatalf("%q", r)
}
dc.expectClosed(2 * time.Second)
e.checkTranscript()
}
// ------------------------------------------------------------------------------------------------------------------------------------
// helpers shared by the splice tests
// ------------------------------------------------------------------------------------------------------------------------------------
func fingerprintOf(der []byte) string {
sum := sha256.Sum256(der)
return hex.EncodeToString(sum[:])
}
func TestLifetime_ARenewFloodIsBusyNotAStreamOfAuthorizeCalls(t *testing.T) {
e := newTestEnv(t, func(c *Config) { c.MinRenewGap = 700 * time.Millisecond })
d := e.newDevice()
dc := e.connectOK(d)
if r := dc.renew(); !strings.HasPrefix(r, "OK ") {
t.Fatalf("first renew: %q", r)
}
for i := 0; i < 5; i++ {
dc.send("RENEW " + e.session(d, nil))
if r := dc.recv(); r != "ERR busy" {
t.Fatalf("a renew right after another: %q", r)
}
}
if n := e.cp.callsFor(d.id, "renew"); n != 1 {
t.Fatalf("the flood reached the control plane %d times", n)
}
time.Sleep(800 * time.Millisecond)
if r := dc.renew(); !strings.HasPrefix(r, "OK ") {
t.Fatalf("a renew after the gap: %q", r)
}
if !e.srv.Online(d.host) {
t.Fatal("the connection did not survive the flood")
}
}
func TestClientHello_ADualStackListenerSeesAnIPv4ClientAsIPv4(t *testing.T) {
e := newTestEnv(t, func(c *Config) { c.HTTPSAddr = "[::]:0" })
d := e.newDevice()
dc := e.connectOK(d)
ds := dc.serve(nil)
ds.noAttach = true
port := e.srv.HTTPSAddr()[strings.LastIndex(e.srv.HTTPSAddr(), ":")+1:]
e.httpsAddr = "127.0.0.1:" + port
sendRaw(t, e, helloOf(d.host, 0, 16000), 0)
eventually(t, "the OPEN", 3*time.Second, func() bool { return ds.openCount() == 1 })
if ip := ds.openList()[0].str("peer_ip"); ip != "127.0.0.1" {
t.Fatalf("peer_ip %q: an IPv4 client must not be written as an IPv4-mapped IPv6 address", ip)
}
e.checkTranscript()
}
func TestAuth_AKeyRotationOfTheControlPlaneIsFollowedAndARevokedKeyIsNotTrusted(t *testing.T) {
e := newTestEnv(t, nil)
d := e.newDevice()
oldKey := e.cpKey
oldPub := oldKey.Public().(ed25519.PublicKey)
newPub, newPriv, _ := ed25519.GenerateKey(rand.Reader)
handshake := func(signer ed25519.PrivateKey, kid string) string {
dc := e.connect(d)
return dc.hello(
func() string { return e.sessionSignedBy(signer, d, func(h, c map[string]any) { h["kid"] = kid }) },
func(s, n string) string { return e.proof(d, s, n, nil) })
}
if r := handshake(oldKey, testKID); !strings.HasPrefix(r, "OK ") {
t.Fatalf("before the rotation: %q", r)
}
// both keys are published during a rotation
e.keys.Set(StaticKeys{testKID: oldPub, "cp-test-2": newPub})
if r := handshake(newPriv, "cp-test-2"); !strings.HasPrefix(r, "OK ") {
t.Fatalf("the new key during the rotation: %q", r)
}
if r := handshake(oldKey, testKID); !strings.HasPrefix(r, "OK ") {
t.Fatalf("the old key during the rotation: %q", r)
}
// the old key is withdrawn: its sessions are refused, the new key's are not
e.keys.Set(StaticKeys{"cp-test-2": newPub})
if r := handshake(oldKey, testKID); r != "ERR auth" {
t.Fatalf("a withdrawn key: %q", r)
}
if r := handshake(newPriv, "cp-test-2"); !strings.HasPrefix(r, "OK ") {
t.Fatalf("the new key after the old one was withdrawn: %q", r)
}
// a key source that vouches for nothing (stale beyond its window) refuses every session
e.keys.Set(nil)
if r := handshake(newPriv, "cp-test-2"); r != "ERR auth" {
t.Fatalf("an empty key set: %q", r)
}
}
files/tunnel/sni.go 278 lines · 7.4 KB
package main
import (
"errors"
"net"
"strings"
"time"
)
// This file reads the CLEARTEXT part of a TLS ClientHello, only to learn the
// server name (SNI) the client asked for. Nothing in this program ever
// decrypts, terminates or alters a client's TLS session: the bytes read here
// are replayed verbatim to the device, which is the real TLS server.
const (
recordTypeHandshake = 22
handshakeClientHello = 1
maxRecordPayload = 1 << 14 // TLS plaintext record limit
maxClientHelloBytes = 64 << 10
maxHelloRecords = 32
// Upper bound on the bytes buffered while looking for the SNI.
maxPeek = maxClientHelloBytes + maxHelloRecords*5 + 4
)
var (
errNotTLS = errors.New("not a TLS handshake record")
errNotClientHello = errors.New("handshake is not a ClientHello")
errHelloMalformed = errors.New("malformed ClientHello")
errHelloTooLarge = errors.New("ClientHello too large")
errBadSNI = errors.New("invalid server name")
)
// helloScan is the outcome of looking at a prefix of the client's bytes.
type helloScan struct {
complete bool // the whole ClientHello message is present
sni string // normalized server name; "" when the extension is absent
need int // when !complete: total buffer length required before rescanning is useful
err error
hello []byte // when complete: the ClientHello's body (the observer reads the groups it offers from it)
}
// scanClientHello inspects buf, the bytes received so far from a client. It
// never panics on any input and never reads outside buf. The handshake message
// may be split across several TLS records (rare, but legal) and across any
// number of TCP reads.
func scanClientHello(buf []byte) helloScan {
if len(buf) == 0 {
return helloScan{need: 1}
}
var hs []byte
off := 0
for rec := 0; ; rec++ {
if rec >= maxHelloRecords {
return helloScan{err: errHelloMalformed}
}
if len(buf)-off >= 1 && buf[off] != recordTypeHandshake {
return helloScan{err: errNotTLS}
}
if len(buf)-off >= 2 && buf[off+1] != 3 {
return helloScan{err: errNotTLS}
}
if len(buf)-off < 5 {
return helloScan{need: off + 5}
}
n := int(buf[off+3])<<8 | int(buf[off+4])
if n == 0 || n > maxRecordPayload {
return helloScan{err: errHelloMalformed}
}
if len(buf)-off-5 < n {
return helloScan{need: off + 5 + n}
}
hs = append(hs, buf[off+5:off+5+n]...)
off += 5 + n
if len(hs) >= 1 && hs[0] != handshakeClientHello {
return helloScan{err: errNotClientHello}
}
if len(hs) >= 4 {
l := int(hs[1])<<16 | int(hs[2])<<8 | int(hs[3])
if l > maxClientHelloBytes {
return helloScan{err: errHelloTooLarge}
}
if len(hs) >= 4+l {
sni, err := parseClientHelloBody(hs[4 : 4+l])
return helloScan{complete: true, sni: sni, err: err, hello: hs[4 : 4+l]}
}
}
}
}
// cursor is a bounds-checked reader over a byte slice.
type cursor struct {
b []byte
bad bool
}
func (c *cursor) take(n int) []byte {
if c.bad || n < 0 || n > len(c.b) {
c.bad = true
return nil
}
out := c.b[:n]
c.b = c.b[n:]
return out
}
func (c *cursor) u8() int {
b := c.take(1)
if b == nil {
return 0
}
return int(b[0])
}
func (c *cursor) u16() int {
b := c.take(2)
if b == nil {
return 0
}
return int(b[0])<<8 | int(b[1])
}
// parseClientHelloBody walks a ClientHello body (the bytes after the 4-byte
// handshake header) and returns the normalized host_name from the
// server_name extension: "" when there is none, errBadSNI when it is present
// but unusable.
func parseClientHelloBody(b []byte) (string, error) {
c := &cursor{b: b}
c.take(2) // legacy_version
c.take(32) // random
c.take(c.u8())
if cs := c.u16(); cs < 2 || cs%2 != 0 {
return "", errHelloMalformed
} else {
c.take(cs)
}
c.take(c.u8()) // compression methods
if c.bad {
return "", errHelloMalformed
}
if len(c.b) == 0 {
return "", nil // no extensions at all
}
extLen := c.u16()
ext := &cursor{b: c.take(extLen)}
if c.bad || len(c.b) != 0 { // the extensions end the message: bytes after them are a hello that lies about its own length
return "", errHelloMalformed
}
found := false
name := ""
for len(ext.b) > 0 {
typ := ext.u16()
data := &cursor{b: ext.take(ext.u16())}
if ext.bad {
return "", errHelloMalformed
}
if typ != 0 { // server_name
continue
}
if found {
return "", errHelloMalformed // duplicate extension
}
found = true
listLen := data.u16()
list := &cursor{b: data.take(listLen)}
if data.bad || len(data.b) != 0 {
return "", errHelloMalformed
}
hostNames := 0
for len(list.b) > 0 {
nameType := list.u8()
raw := list.take(list.u16())
if list.bad {
return "", errHelloMalformed
}
if nameType != 0 {
continue
}
hostNames++
if hostNames > 1 {
return "", errHelloMalformed
}
n, ok := normalizeName(string(raw))
if !ok {
return "", errBadSNI
}
name = n
}
}
return name, nil
}
// readClientHello reads from c until the ClientHello is complete (or invalid,
// too large, or the deadline passes). It returns EVERY byte it read from the
// connection, which the caller must forward to the device first.
func readClientHello(c net.Conn, timeout time.Duration) (peeked []byte, sni string, err error) {
_ = c.SetReadDeadline(time.Now().Add(timeout))
defer func() { _ = c.SetReadDeadline(time.Time{}) }()
buf := make([]byte, 0, 2048)
need := 1
for {
for len(buf) >= need {
res := scanClientHello(buf)
if res.err != nil {
return buf, "", res.err
}
if res.complete {
return buf, res.sni, nil
}
need = res.need
if need > maxPeek {
return buf, "", errHelloTooLarge
}
}
if len(buf) == cap(buf) {
next := 2 * cap(buf)
if next > maxPeek {
next = maxPeek
}
if next <= len(buf) {
return buf, "", errHelloTooLarge
}
grown := make([]byte, len(buf), next)
copy(grown, buf)
buf = grown
}
n, rerr := c.Read(buf[len(buf):cap(buf)])
buf = buf[:len(buf)+n]
if rerr != nil {
if n > 0 && len(buf) >= need {
continue // decide on what we have before reporting the error
}
return buf, "", rerr
}
}
}
// normalizeName is the one way a server name becomes a routing key, for an SNI and for a Host header alike: lower case, at most one trailing dot
// removed, then strict LDH. A name the table could hold under two spellings, or that a resolver would read differently from the node, is refused:
// letters, digits and hyphens only (no underscore), labels of 1..63 bytes that neither start nor end with a hyphen, at most 253 bytes in all, a last
// label that is not all digits (so no dotted-decimal address in disguise) and not an IP literal. The device's own name always passes: it is one
// label under a zone.
func normalizeName(s string) (string, bool) {
b := []byte(s)
for i, ch := range b {
if ch >= 'A' && ch <= 'Z' {
b[i] = ch + ('a' - 'A')
}
}
s = string(b)
if len(s) > 1 && s[len(s)-1] == '.' {
s = s[:len(s)-1]
}
if s == "" || len(s) > 253 || net.ParseIP(s) != nil {
return "", false
}
start := 0
for i := 0; i <= len(s); i++ {
if i < len(s) && s[i] != '.' {
ch := s[i]
if !(ch >= 'a' && ch <= 'z' || ch >= '0' && ch <= '9' || ch == '-') {
return "", false
}
continue
}
label := s[start:i]
if len(label) == 0 || len(label) > 63 || label[0] == '-' || label[len(label)-1] == '-' {
return "", false
}
if i == len(s) && strings.Trim(label, "0123456789") == "" {
return "", false
}
start = i + 1
}
return s, true
}
files/tunnel/sni_test.go 412 lines · 15.4 KB
package main
import (
"bytes"
"crypto/tls"
"errors"
"io"
mrand "math/rand"
"net"
"os"
"strings"
"testing"
"testing/iotest"
"time"
)
// captureClientHello returns the first TLS record a real crypto/tls client
// sends (its ClientHello), exactly as it appears on the wire.
func captureClientHello(t testing.TB, cfg *tls.Config) []byte {
t.Helper()
c1, c2 := net.Pipe()
defer c1.Close()
defer c2.Close()
go func() { _ = tls.Client(c1, cfg).Handshake() }()
_ = c2.SetReadDeadline(time.Now().Add(5 * time.Second))
hdr := make([]byte, 5)
if _, err := io.ReadFull(c2, hdr); err != nil {
t.Fatal(err)
}
rec := make([]byte, int(hdr[3])<<8|int(hdr[4]))
if _, err := io.ReadFull(c2, rec); err != nil {
t.Fatal(err)
}
return append(hdr, rec...)
}
func realHello(t testing.TB, serverName string) []byte {
return captureClientHello(t, &tls.Config{ServerName: serverName, InsecureSkipVerify: true, NextProtos: []string{"http/1.1"}})
}
// refragment rewraps the handshake payload of a single-record ClientHello
// into records of at most n payload bytes each.
func refragment(hello []byte, n int) []byte {
payload := hello[5:]
var out []byte
for len(payload) > 0 {
k := min(n, len(payload))
out = append(out, 22, 3, 3, byte(k>>8), byte(k))
out = append(out, payload[:k]...)
payload = payload[k:]
}
return out
}
func TestScanSNI_RealClientHellos(t *testing.T) {
cases := []struct{ in, want string }{
{"dev1.example.test", "dev1.example.test"},
{"Dev1.Example.TEST", "dev1.example.test"}, // normalized
{"nodo-abc123.example.test", "nodo-abc123.example.test"},
{"", ""}, // no SNI at all
{"127.0.0.1", ""}, // Go omits SNI for IP literals
}
for _, tc := range cases {
t.Run(tc.in, func(t *testing.T) {
res := scanClientHello(realHello(t, tc.in))
if res.err != nil || !res.complete || res.sni != tc.want {
t.Fatalf("got %+v, want sni %q", res, tc.want)
}
})
}
}
// Modern clients (Chrome, Go 1.24+, OpenSSL 3.5) offer the X25519MLKEM768
// hybrid key share, which makes the ClientHello well over a kilobyte.
func TestScanSNI_LargePostQuantumClientHello(t *testing.T) {
hello := captureClientHello(t, &tls.Config{
ServerName: "dev1.example.test", InsecureSkipVerify: true,
CurvePreferences: []tls.CurveID{tls.X25519MLKEM768, tls.X25519},
})
if len(hello) < 1200 {
t.Fatalf("expected a large (ML-KEM) hello, got %d bytes", len(hello))
}
if res := scanClientHello(hello); res.err != nil || !res.complete || res.sni != "dev1.example.test" {
t.Fatalf("%+v", res)
}
// ... split in half across two records, and arriving a few bytes at a time.
if res := scanClientHello(refragment(hello, len(hello)/2)); res.err != nil || res.sni != "dev1.example.test" {
t.Fatalf("two records: %+v", res)
}
peeked, sni, err := readClientHello(byteConn{r: iotest.OneByteReader(bytes.NewReader(hello))}, time.Second)
if err != nil || sni != "dev1.example.test" || !bytes.Equal(peeked, hello) {
t.Fatalf("one-byte reads: sni=%q err=%v", sni, err)
}
}
// A padded hello close to the 16 KiB record limit (RFC 7685 padding, ECH).
func TestScanSNI_HugePaddedHello(t *testing.T) {
pad := make([]byte, 14000)
ext := append([]byte{0, 21, byte(len(pad) >> 8), byte(len(pad))}, pad...) // padding extension
ext = append(ext, sniExt("dev1.example.test")...)
hello := buildHello(ext, true)
if len(hello) < 14000 || len(hello) > maxRecordPayload {
t.Fatalf("test hello is %d bytes", len(hello))
}
peeked, sni, err := readClientHello(byteConn{r: bytes.NewReader(hello)}, time.Second)
if err != nil || sni != "dev1.example.test" || !bytes.Equal(peeked, hello) {
t.Fatalf("sni=%q err=%v", sni, err)
}
}
func TestScanSNI_EveryPrefixNeedsMore(t *testing.T) {
hello := realHello(t, "dev1.example.test")
for n := 0; n < len(hello); n++ {
res := scanClientHello(hello[:n])
if res.err != nil || res.complete || res.need <= n {
t.Fatalf("prefix %d/%d: got %+v", n, len(hello), res)
}
}
if res := scanClientHello(hello); !res.complete || res.sni != "dev1.example.test" {
t.Fatalf("full: %+v", res)
}
}
func TestScanSNI_HelloSplitAcrossRecords(t *testing.T) {
hello := realHello(t, "dev1.example.test")
payload := len(hello) - 5
for _, n := range []int{payload/maxHelloRecords + 1, 100, 300, 1000, payload} {
frag := refragment(hello, n)
res := scanClientHello(frag)
if res.err != nil || !res.complete || res.sni != "dev1.example.test" {
t.Fatalf("records of %d bytes: %+v", n, res)
}
// no decision before the last byte
for i := 0; i < len(frag); i++ {
if r := scanClientHello(frag[:i]); r.complete || r.err != nil {
t.Fatalf("records of %d bytes: decided early at %d/%d: %+v", n, i, len(frag), r)
}
}
}
// Absurdly fragmented hellos (more records than the cap) are refused.
if res := scanClientHello(refragment(hello, 1)); !errors.Is(res.err, errHelloMalformed) {
t.Fatalf("1-byte records: %+v", res)
}
}
func TestScanSNI_TrailingBytesAreIgnored(t *testing.T) {
hello := realHello(t, "dev1.example.test")
res := scanClientHello(append(append([]byte(nil), hello...), []byte("\x17\x03\x03\x00\x05trail")...))
if res.err != nil || !res.complete || res.sni != "dev1.example.test" {
t.Fatalf("%+v", res)
}
}
// buildHello assembles a ClientHello record from parts, for hostile cases.
func buildHello(extensions []byte, withExt bool) []byte {
var body []byte
body = append(body, 3, 3) // legacy_version
body = append(body, make([]byte, 32)...) // random
body = append(body, 0) // session id len
body = append(body, 0, 2, 0x13, 0x01) // one cipher suite
body = append(body, 1, 0) // compression
if withExt {
body = append(body, byte(len(extensions)>>8), byte(len(extensions)))
body = append(body, extensions...)
}
hs := append([]byte{1, byte(len(body) >> 16), byte(len(body) >> 8), byte(len(body))}, body...)
return append([]byte{22, 3, 1, byte(len(hs) >> 8), byte(len(hs))}, hs...)
}
func sniExt(names ...string) []byte {
var list []byte
for _, n := range names {
list = append(list, 0, byte(len(n)>>8), byte(len(n)))
list = append(list, n...)
}
data := append([]byte{byte(len(list) >> 8), byte(len(list))}, list...)
return append([]byte{0, 0, byte(len(data) >> 8), byte(len(data))}, data...)
}
func TestScanSNI_HandBuiltHellos(t *testing.T) {
good := scanClientHello(buildHello(sniExt("a.example.test"), true))
if good.err != nil || good.sni != "a.example.test" {
t.Fatalf("baseline: %+v", good)
}
noExt := scanClientHello(buildHello(nil, false))
if noExt.err != nil || !noExt.complete || noExt.sni != "" {
t.Fatalf("no extensions: %+v", noExt)
}
otherExtOnly := scanClientHello(buildHello([]byte{0, 43, 0, 3, 2, 3, 4}, true))
if otherExtOnly.err != nil || otherExtOnly.sni != "" {
t.Fatalf("other extension only: %+v", otherExtOnly)
}
}
func TestScanSNI_Rejects(t *testing.T) {
dupExt := append(sniExt("a.example.test"), sniExt("b.example.test")...)
rec := func(typ byte, ver byte, payload []byte) []byte {
return append([]byte{typ, 3, ver, byte(len(payload) >> 8), byte(len(payload))}, payload...)
}
var tooManyRecords []byte
for _, b := range append([]byte{1, 0, 0, 200}, make([]byte, 40)...) {
tooManyRecords = append(tooManyRecords, rec(22, 1, []byte{b})...)
}
cases := []struct {
name string
in []byte
want error
}{
{"http on 443", []byte("GET / HTTP/1.1\r\nHost: x\r\n\r\n"), errNotTLS},
{"ssl2-style", []byte{0x80, 0x2e, 0x01, 0x00, 0x02}, errNotTLS},
{"application data record", rec(23, 3, []byte{1, 2, 3}), errNotTLS},
{"bad record major version", append([]byte{22, 7, 3, 0, 4}, 1, 0, 0, 0), errNotTLS},
{"zero length record", []byte{22, 3, 1, 0, 0}, errHelloMalformed},
{"oversize record", []byte{22, 3, 1, 0xff, 0xff}, errHelloMalformed},
{"server hello", rec(22, 3, []byte{2, 0, 0, 0}), errNotClientHello},
{"hello longer than cap", rec(22, 3, []byte{1, 0xff, 0xff, 0xff}), errHelloTooLarge},
{"body shorter than its fields", rec(22, 3, []byte{1, 0, 0, 5, 3, 3, 0, 0, 0}), errHelloMalformed},
{"duplicate server_name extension", buildHello(dupExt, true), errHelloMalformed},
{"two host_names in one extension", buildHello(sniExt("a.example.test", "b.example.test"), true), errHelloMalformed},
{"space in name", buildHello(sniExt("a b.example.test"), true), errBadSNI},
{"NUL in name", buildHello(sniExt("a\x00b.example.test"), true), errBadSNI},
{"slash in name", buildHello(sniExt("a/b.example.test"), true), errBadSNI},
{"empty name", buildHello(sniExt(""), true), errBadSNI},
{"ip literal name", buildHello(sniExt("192.0.2.1"), true), errBadSNI},
{"label over 63", buildHello(sniExt(strings.Repeat("a", 64)+".example.test"), true), errBadSNI},
{"empty label", buildHello(sniExt("a..example.test"), true), errBadSNI},
{"leading dot", buildHello(sniExt(".example.test"), true), errBadSNI},
{"name over 253", buildHello(sniExt(strings.Repeat("abcdefgh.", 30)+"test"), true), errBadSNI},
{"extension length overruns", buildHello([]byte{0, 0, 0xff, 0xff, 1}, true), errHelloMalformed},
{"too many records", tooManyRecords, errHelloMalformed},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
res := scanClientHello(tc.in)
if !errors.Is(res.err, tc.want) {
t.Fatalf("got %+v, want error %v", res, tc.want)
}
})
}
}
// Mutating a valid hello in every way must never panic or run out of bounds.
func TestScanSNI_NeverPanicsOnMutations(t *testing.T) {
hello := realHello(t, "dev1.example.test")
rng := mrand.New(mrand.NewSource(1))
for i := 0; i < 30000; i++ {
b := append([]byte(nil), hello...)
for k := rng.Intn(4); k >= 0; k-- {
b[rng.Intn(len(b))] = byte(rng.Intn(256))
}
if rng.Intn(3) == 0 {
b = b[:rng.Intn(len(b)+1)]
}
res := scanClientHello(b)
if res.complete && res.err == nil && res.sni != "" {
if n, ok := normalizeName(res.sni); !ok || n != res.sni {
t.Fatalf("returned an unnormalized name %q", res.sni)
}
}
}
}
func FuzzScanClientHello(f *testing.F) {
f.Add(realHello(f, "dev1.example.test"))
f.Add(realHello(f, ""))
f.Add(buildHello(sniExt("a.example.test"), true))
f.Add([]byte("GET / HTTP/1.1\r\n\r\n"))
f.Fuzz(func(t *testing.T, data []byte) {
res := scanClientHello(data)
if res.complete && res.err == nil && res.sni != "" {
if n, ok := normalizeName(res.sni); !ok || n != res.sni {
t.Fatalf("returned an unnormalized name %q", res.sni)
}
}
if !res.complete && res.err == nil && res.need <= len(data) {
t.Fatalf("need %d not beyond len %d", res.need, len(data))
}
})
}
// byteConn is a net.Conn that serves a reader (for readClientHello tests).
type byteConn struct {
net.Conn
r io.Reader
}
func (b byteConn) Read(p []byte) (int, error) { return b.r.Read(p) }
func (b byteConn) SetReadDeadline(time.Time) error { return nil }
func TestReadClientHello_OneByteReads(t *testing.T) {
hello := realHello(t, "dev1.example.test")
trail := []byte("EXTRA")
in := append(append([]byte(nil), hello...), trail...)
peeked, sni, err := readClientHello(byteConn{r: iotest.OneByteReader(bytes.NewReader(in))}, time.Second)
if err != nil || sni != "dev1.example.test" {
t.Fatalf("sni=%q err=%v", sni, err)
}
if !bytes.Equal(peeked, hello) { // one-byte reads cannot over-read past the hello
t.Fatalf("peeked %d bytes, want exactly the %d-byte hello", len(peeked), len(hello))
}
}
func TestReadClientHello_ReturnsEveryByteRead(t *testing.T) {
hello := realHello(t, "dev1.example.test")
in := append(append([]byte(nil), hello...), []byte("COALESCED-AFTER-HELLO")...)
peeked, sni, err := readClientHello(byteConn{r: bytes.NewReader(in)}, time.Second)
if err != nil || sni != "dev1.example.test" {
t.Fatalf("sni=%q err=%v", sni, err)
}
if !bytes.Equal(peeked, in) { // everything read must be replayed to the device
t.Fatalf("peeked %d bytes, want all %d", len(peeked), len(in))
}
}
func TestReadClientHello_SplitReadsOverPipe(t *testing.T) {
hello := realHello(t, "dev1.example.test")
c1, c2 := net.Pipe()
defer c1.Close()
defer c2.Close()
go func() {
for i := 0; i < len(hello); i += 3 {
_, _ = c1.Write(hello[i:min(i+3, len(hello))])
time.Sleep(200 * time.Microsecond)
}
}()
peeked, sni, err := readClientHello(c2, 5*time.Second)
if err != nil || sni != "dev1.example.test" || !bytes.Equal(peeked, hello) {
t.Fatalf("sni=%q err=%v peeked=%d/%d", sni, err, len(peeked), len(hello))
}
}
func TestReadClientHello_MissingSNIIsNotAnError(t *testing.T) {
peeked, sni, err := readClientHello(byteConn{r: bytes.NewReader(realHello(t, ""))}, time.Second)
if err != nil || sni != "" || len(peeked) == 0 {
t.Fatalf("sni=%q err=%v", sni, err)
}
}
func TestReadClientHello_FailureModes(t *testing.T) {
t.Run("timeout", func(t *testing.T) {
c1, c2 := net.Pipe()
defer c1.Close()
defer c2.Close()
_, _, err := readClientHello(c2, 100*time.Millisecond)
if !errors.Is(err, os.ErrDeadlineExceeded) {
t.Fatalf("got %v", err)
}
})
t.Run("eof before hello", func(t *testing.T) {
hello := realHello(t, "x.example.test")
_, _, err := readClientHello(byteConn{r: bytes.NewReader(hello[:20])}, time.Second)
if !errors.Is(err, io.EOF) {
t.Fatalf("got %v", err)
}
})
t.Run("not tls fails fast", func(t *testing.T) {
_, _, err := readClientHello(byteConn{r: strings.NewReader("GET / HTTP/1.1\r\n")}, time.Second)
if !errors.Is(err, errNotTLS) {
t.Fatalf("got %v", err)
}
})
t.Run("stream of records beyond the cap", func(t *testing.T) {
endless := io.MultiReader(bytes.NewReader([]byte{22, 3, 1, 0x3f, 0xff, 1, 0, 0xff, 0xff}), iotest.ErrReader(io.ErrUnexpectedEOF))
_, _, err := readClientHello(byteConn{r: endless}, time.Second)
if err == nil {
t.Fatal("expected an error")
}
})
}
func TestNormalizeName(t *testing.T) {
ok := map[string]string{
"Example.COM": "example.com", "a.b.c.": "a.b.c",
"nodo-abc123.example.test": "nodo-abc123.example.test",
"localhost": "localhost", "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test",
"PSRHZ6N5WPZBM6YFKNGO.Devices.ALL-E2E.test.": "psrhz6n5wpzbm6yfkngo.devices.all-e2e.test",
"xn--bcher-kva.example": "xn--bcher-kva.example",
}
for in, want := range ok {
if got, good := normalizeName(in); !good || got != want {
t.Errorf("%q -> %q,%v want %q", in, got, good, want)
}
}
long63, long64 := strings.Repeat("a", 63), strings.Repeat("a", 64)
for _, in := range []string{
"", ".", "..", "a..b", "a.b..", ".a.b", "a b", "a/b", "::1", "127.0.0.1", "[::1]", "é.example.test", "a\x00b", "a\tb", "a\nb", "a.b\n",
"x_y.example.test", "_acme-challenge.example.test", "-a.example.test", "a-.example.test", "a.-b.test", "a.b-.test", "a*.example.test",
"192.0.2.4", "1.2.3", "0x7f.1", "a.123", "999", long64 + ".test", strings.Repeat(long63+".", 4) + "test", "a.b.c:443", "a" + "@b.test", "a b.test", "ａ.test",
} {
if got, good := normalizeName(in); good {
t.Errorf("%q should be rejected, got %q", in, got)
}
}
if _, good := normalizeName(long63 + ".test"); !good {
t.Error("a 63-byte label is valid")
}
}
func TestScanSNI_BytesAfterTheExtensionsAreAHelloThatLiesAboutItsLength(t *testing.T) {
hello := buildHello(sniExt("a.example.test"), true)
// grow the handshake message by two bytes that no field accounts for
lying := append([]byte(nil), hello...)
lying = append(lying, 0xde, 0xad)
hsLen := int(lying[6])<<16 | int(lying[7])<<8 | int(lying[8])
hsLen += 2
lying[6], lying[7], lying[8] = byte(hsLen>>16), byte(hsLen>>8), byte(hsLen)
recLen := int(lying[3])<<8 | int(lying[4]) + 2
lying[3], lying[4] = byte(recLen>>8), byte(recLen)
if res := scanClientHello(lying); res.err == nil {
t.Fatalf("accepted: %+v", res)
}
}
files/tunnel/splice_test.go 577 lines · 17.1 KB
package main
import (
"bufio"
"bytes"
"crypto/rand"
"crypto/sha256"
"crypto/tls"
"fmt"
"io"
"net"
"net/http"
"strings"
"sync"
"testing"
"time"
)
// ------------------------------------------------------------------------------------------------------------------------------------
// Blindness: the node forwards ciphertext and never the plaintext.
// ------------------------------------------------------------------------------------------------------------------------------------
func TestBlindSplice_NodeNeverSeesPlaintext(t *testing.T) {
e := newTestEnv(t, nil)
d := e.newDevice()
dc := e.connectOK(d)
ds := dc.serve(nil)
c, err := e.clientTLS(d)
if err != nil {
t.Fatal(err)
}
defer c.Close()
clientAddr := c.LocalAddr().String()
// The client verified the DEVICE's certificate (issued by the device's CA for the device's name). It is not the node's own certificate.
st := c.ConnectionState()
if fingerprintOf(st.PeerCertificates[0].Raw) != fingerprintOf(d.cert.Certificate[0]) {
t.Fatal("client did not reach the device's TLS server")
}
if st.NegotiatedProtocol != "http/1.1" {
t.Fatalf("ALPN = %q, want http/1.1", st.NegotiatedProtocol)
}
body := []byte(strings.Repeat("A", 100) + marker + strings.Repeat("B", 5000))
resp, err := postOver(c, d.host, body)
if err != nil {
t.Fatal(err)
}
// Teeth: the endpoints really exchanged the marker in the clear.
if !bytes.Contains(resp, []byte(marker)) {
t.Fatal("client did not read the marker back from the device")
}
_ = c.Close()
e.waitIdle()
seen := ds.seen()
if len(seen) != 1 || !bytes.Contains(seen[0].body, []byte(marker)) {
t.Fatalf("the device did not receive the marker in plaintext: %+v", seen)
}
opens := ds.openList()
if len(opens) != 1 {
t.Fatalf("expected exactly one stream, got %v", opens)
}
// What the node could see: every byte it forwarded (both directions, including the peeked ClientHello) and every log line.
tapped := e.tapped(opens[0].str("stream_id"))
if len(tapped) < len(body)+len(resp) {
t.Fatalf("tap saw only %d bytes; the stream carried at least %d", len(tapped), len(body)+len(resp))
}
for name, hay := range map[string][]byte{
"forwarded bytes (this stream)": tapped,
"forwarded bytes (all)": e.tappedAll(),
"logs": []byte(e.logs.String()),
} {
if bytes.Contains(hay, []byte(marker)) {
t.Fatalf("node-visible %s contain the plaintext marker", name)
}
if bytes.Contains(hay, []byte(strings.Repeat("A", 100))) || bytes.Contains(hay, []byte(strings.Repeat("B", 100))) {
t.Fatalf("node-visible %s contain plaintext filler", name)
}
}
// What it saw is TLS: a ClientHello first, then encrypted application data.
if !bytes.HasPrefix(tapped, []byte{0x16, 0x03}) {
t.Fatalf("first forwarded bytes are not a TLS handshake record: % x", tapped[:8])
}
if !bytes.Contains(tapped, []byte{0x17, 0x03, 0x03}) {
t.Fatal("no TLS application-data record seen")
}
// Logs carry only metadata (sizes, ids, times, reasons): no name, no address, no identity of a device.
for _, want := range []string{`"stream_open"`, `"stream_close"`, `"up_bytes"`, `"down_bytes"`} {
if !strings.Contains(e.logs.String(), want) {
t.Fatalf("logs lack %s:\n%s", want, e.logs.String())
}
}
for name, secret := range map[string]string{"host": d.host, "label": d.label, "device id": d.id, "client address": clientAddr, "session": dc.session} {
if strings.Contains(e.logs.String(), secret) {
t.Fatalf("the logs name the %s", name)
}
}
e.checkTranscript()
}
// ------------------------------------------------------------------------------------------------------------------------------------
// Splice correctness
// ------------------------------------------------------------------------------------------------------------------------------------
func TestSplice_LargeBodiesBothWays(t *testing.T) {
e := newTestEnv(t, nil)
d := e.newDevice()
dc := e.connectOK(d)
dc.serve(nil)
body := make([]byte, 6<<20)
if _, err := rand.Read(body); err != nil {
t.Fatal(err)
}
c, err := e.clientTLS(d)
if err != nil {
t.Fatal(err)
}
defer c.Close()
resp, err := postOver(c, d.host, body)
if err != nil {
t.Fatal(err)
}
if len(resp) != len(body)+5 || sha256.Sum256(resp[5:]) != sha256.Sum256(body) {
t.Fatalf("body corrupted or truncated: got %d bytes, want %d", len(resp), len(body)+5)
}
_ = c.Close()
e.waitIdle()
if n := e.logs.Count(`"reason":"closed"`); n < 1 {
t.Fatalf("stream did not close cleanly:\n%s", e.logs.String())
}
}
func TestSplice_ManyConcurrentStreams(t *testing.T) {
e := newTestEnv(t, nil)
d := e.newDevice()
dc := e.connectOK(d)
ds := dc.serve(nil)
const n = 30
var wg sync.WaitGroup
errs := make(chan error, n)
for i := 0; i < n; i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
c, err := e.clientTLS(d)
if err != nil {
errs <- err
return
}
defer c.Close()
body := append(make([]byte, 20000), fmt.Sprintf("stream-%d", i)...)
if _, err := rand.Read(body[:20000]); err != nil {
errs <- err
return
}
resp, err := postOver(c, d.host, body)
if err != nil {
errs <- err
return
}
if !bytes.Equal(resp, append([]byte("echo:"), body...)) {
errs <- fmt.Errorf("stream %d: wrong echo", i)
}
}(i)
}
wg.Wait()
close(errs)
for err := range errs {
t.Error(err)
}
e.waitIdle()
ids := map[string]bool{}
for _, m := range ds.openList() {
ids[m.str("stream_id")] = true
}
if len(ids) != n {
t.Fatalf("expected %d distinct stream ids, got %d", n, len(ids))
}
e.checkTranscript()
}
func TestSplice_ClientHelloArrivesInTinyPieces(t *testing.T) {
e := newTestEnv(t, nil)
d := e.newDevice()
dc := e.connectOK(d)
dc.serve(nil)
tcp, err := net.Dial("tcp", e.httpsAddr)
if err != nil {
t.Fatal(err)
}
c := tls.Client(chunkConn{Conn: tcp, size: 5, delay: time.Millisecond}, &tls.Config{ServerName: d.host, RootCAs: d.pool, NextProtos: []string{"http/1.1"}})
defer c.Close()
_ = c.SetDeadline(time.Now().Add(20 * time.Second))
if err := c.Handshake(); err != nil {
t.Fatal(err)
}
resp, err := postOver(c, d.host, []byte("tiny-pieces-"+marker))
if err != nil || !bytes.HasSuffix(resp, []byte(marker)) {
t.Fatalf("resp=%q err=%v", resp, err)
}
}
func TestSplice_ClientHalfClosesButStillReadsResponse(t *testing.T) {
e := newTestEnv(t, nil)
d := e.newDevice()
dc := e.connectOK(d)
dc.serve(nil)
tcp, err := net.Dial("tcp", e.httpsAddr)
if err != nil {
t.Fatal(err)
}
c := tls.Client(tcp, &tls.Config{ServerName: d.host, RootCAs: d.pool, NextProtos: []string{"http/1.1"}})
defer c.Close()
_ = c.SetDeadline(time.Now().Add(20 * time.Second))
body := []byte("half-close-" + marker)
req, _ := http.NewRequest("POST", "https://"+d.host+"/mcp", bytes.NewReader(body))
req.ContentLength = int64(len(body))
if err := req.Write(c); err != nil {
t.Fatal(err)
}
// FIN toward the node right after the request: the device's answer must still make it back.
if err := tcp.(*net.TCPConn).CloseWrite(); err != nil {
t.Fatal(err)
}
resp, err := http.ReadResponse(bufio.NewReader(c), req)
if err != nil {
t.Fatal(err)
}
got, _ := io.ReadAll(resp.Body)
if !bytes.HasSuffix(got, []byte(marker)) {
t.Fatalf("response lost after client half-close: %q", got)
}
}
func deviceTLSServer(d *testDevice, c net.Conn) *tls.Conn {
return tls.Server(c, &tls.Config{Certificates: []tls.Certificate{d.cert}, NextProtos: []string{"http/1.1"}})
}
func TestSplice_DeviceResetMidResponseClosesClientPromptly(t *testing.T) {
e := newTestEnv(t, nil)
d := e.newDevice()
dc := e.connectOK(d)
started := make(chan struct{}, 1)
dc.serve(func(c net.Conn) {
tc := deviceTLSServer(d, c)
if tc.Handshake() != nil {
return
}
req, err := http.ReadRequest(bufio.NewReader(tc))
if err != nil {
return
}
_, _ = io.Copy(io.Discard, req.Body)
_, _ = io.WriteString(tc, "HTTP/1.1 200 OK\r\nContent-Length: 100000\r\n\r\nhalf")
started <- struct{}{}
// Abort the tunnel connection with a TCP reset.
raw := c.(*tls.Conn).NetConn().(*net.TCPConn)
_ = raw.SetLinger(0)
_ = raw.Close()
})
c, err := e.clientTLS(d)
if err != nil {
t.Fatal(err)
}
defer c.Close()
if _, err = postOver(c, d.host, []byte("x")); err == nil {
t.Fatal("client should see the truncated response as an error")
}
<-started
e.waitIdle()
if e.logs.Count(`"reason":"conn_error"`) == 0 {
t.Fatalf("expected conn_error:\n%s", e.logs.String())
}
}
func TestSplice_ClientResetClosesDevicePromptly(t *testing.T) {
e := newTestEnv(t, nil)
d := e.newDevice()
dc := e.connectOK(d)
handlerDone := make(chan error, 1)
dc.serve(func(c net.Conn) {
tc := deviceTLSServer(d, c)
if err := tc.Handshake(); err != nil {
handlerDone <- err
return
}
_ = tc.SetReadDeadline(time.Now().Add(10 * time.Second))
_, err := io.Copy(io.Discard, tc) // blocks until the stream dies
handlerDone <- err
})
tcp, err := net.Dial("tcp", e.httpsAddr)
if err != nil {
t.Fatal(err)
}
c := tls.Client(tcp, &tls.Config{ServerName: d.host, RootCAs: d.pool})
if err := c.Handshake(); err != nil {
t.Fatal(err)
}
_ = tcp.(*net.TCPConn).SetLinger(0)
_ = tcp.Close() // RST
select {
case <-handlerDone:
case <-time.After(5 * time.Second):
t.Fatal("device side was not closed after the client vanished")
}
e.waitIdle()
}
// ------------------------------------------------------------------------------------------------------------------------------------
// :443 routing decisions
// ------------------------------------------------------------------------------------------------------------------------------------
func TestSNIRouting_Rejections(t *testing.T) {
e := newTestEnv(t, func(c *Config) { c.HelloTimeout = 400 * time.Millisecond })
d := e.newDevice()
dc := e.connectOK(d)
ds := dc.serve(nil)
dialTLS := func(serverName string) error {
tcp, err := net.Dial("tcp", e.httpsAddr)
if err != nil {
t.Fatal(err)
}
defer tcp.Close()
_ = tcp.SetDeadline(time.Now().Add(5 * time.Second))
return tls.Client(tcp, &tls.Config{ServerName: serverName, InsecureSkipVerify: true}).Handshake()
}
t.Run("no SNI is closed with an alert", func(t *testing.T) {
err := dialTLS("")
if err == nil || !strings.Contains(err.Error(), "unrecognized name") {
t.Fatalf("err = %v", err)
}
if e.logs.Count(`"reason":"no_sni"`) != 1 {
t.Fatalf("logs:\n%s", e.logs.String())
}
})
t.Run("unknown host", func(t *testing.T) {
err := dialTLS("nobody.example.test")
if err == nil || !strings.Contains(err.Error(), "unrecognized name") {
t.Fatalf("err = %v", err)
}
})
t.Run("unusable SNI value", func(t *testing.T) {
out := rawExchange(t, e.httpsAddr, string(buildHello(sniExt("192.0.2.3"), true)))
if out != "\x15\x03\x03\x00\x02\x02\x70" {
t.Fatalf("response = %q", out)
}
if e.logs.Count(`"reason":"bad_sni"`) != 1 {
t.Fatalf("logs:\n%s", e.logs.String())
}
})
t.Run("plain HTTP on 443 is just closed", func(t *testing.T) {
if out := rawExchange(t, e.httpsAddr, "GET / HTTP/1.1\r\nHost: "+d.host+"\r\n\r\n"); out != "" {
t.Fatalf("response = %q", out)
}
if e.logs.Count(`"reason":"not_tls"`) != 1 {
t.Fatalf("logs:\n%s", e.logs.String())
}
})
t.Run("silent client times out", func(t *testing.T) {
c, err := net.Dial("tcp", e.httpsAddr)
if err != nil {
t.Fatal(err)
}
defer c.Close()
_ = c.SetReadDeadline(time.Now().Add(5 * time.Second))
start := time.Now()
if n, err := c.Read(make([]byte, 1)); n != 0 || err == nil {
t.Fatalf("n=%d err=%v", n, err)
}
if time.Since(start) > 3*time.Second {
t.Fatal("hello timeout not enforced")
}
eventually(t, "hello_timeout log", 2*time.Second, func() bool { return e.logs.Count(`"reason":"hello_timeout"`) == 1 })
})
if n := ds.openCount(); n != 0 {
t.Fatalf("the device was asked to open %d streams for traffic that should never reach it", n)
}
}
func TestSNIRouting_DeviceOffline(t *testing.T) {
e := newTestEnv(t, nil)
d := e.newDevice()
// No device connected yet.
if _, err := e.clientTLS(d); err == nil || !strings.Contains(err.Error(), "unrecognized name") {
t.Fatalf("err = %v", err)
}
// Online, works; offline again after the control connection ends.
dc := e.connectOK(d)
dc.serve(nil)
c, err := e.clientTLS(d)
if err != nil {
t.Fatal(err)
}
_ = c.Close()
_ = dc.c.Close()
eventually(t, "device offline", 3*time.Second, func() bool { return !e.srv.Online(d.host) })
if _, err := e.clientTLS(d); err == nil {
t.Fatal("handshake should fail while the device is offline")
}
e.checkTranscript()
}
// ------------------------------------------------------------------------------------------------------------------------------------
// Limits, timeouts, shutdown
// ------------------------------------------------------------------------------------------------------------------------------------
func holdHandler(d *testDevice, release <-chan struct{}) func(net.Conn) {
return func(c net.Conn) {
tc := deviceTLSServer(d, c)
if tc.Handshake() != nil {
return
}
select {
case <-release:
case <-time.After(15 * time.Second):
}
}
}
func TestLimits_PerHostAndGlobal(t *testing.T) {
t.Run("per host", func(t *testing.T) {
release := make(chan struct{})
e := newTestEnv(t, func(c *Config) { c.MaxStreamsPerHost = 2 })
d := e.newDevice()
dc := e.connectOK(d)
dc.serve(holdHandler(d, release))
c1, err1 := e.clientTLS(d)
c2, err2 := e.clientTLS(d)
if err1 != nil || err2 != nil {
t.Fatalf("first two streams must work: %v %v", err1, err2)
}
if _, err := e.clientTLS(d); err == nil || !strings.Contains(err.Error(), "internal error") {
t.Fatalf("third stream must be refused, got %v", err)
}
if e.logs.Count(`"reason":"host_limit"`) != 1 {
t.Fatalf("logs:\n%s", e.logs.String())
}
close(release)
c1.Close()
c2.Close()
e.waitIdle()
e.srv.routes.mu.Lock()
leaked := e.srv.routes.byHost[d.host].streams
e.srv.routes.mu.Unlock()
if leaked != 0 {
t.Fatalf("per-host counter leaked: %d", leaked)
}
})
t.Run("global", func(t *testing.T) {
release := make(chan struct{})
e := newTestEnv(t, func(c *Config) { c.MaxStreams = 1 })
d := e.newDevice()
dc := e.connectOK(d)
dc.serve(holdHandler(d, release))
c1, err := e.clientTLS(d)
if err != nil {
t.Fatal(err)
}
if _, err := e.clientTLS(d); err == nil {
t.Fatal("second stream must be refused")
}
close(release)
c1.Close()
e.waitIdle()
if e.logs.Count(`"reason":"global_limit"`) != 1 {
t.Fatalf("logs:\n%s", e.logs.String())
}
})
}
func TestLimits_StreamByteCap(t *testing.T) {
e := newTestEnv(t, func(c *Config) { c.MaxStreamBytes = 100_000 })
d := e.newDevice()
dc := e.connectOK(d)
dc.serve(nil)
c, err := e.clientTLS(d)
if err != nil {
t.Fatal(err)
}
defer c.Close()
if _, err := postOver(c, d.host, make([]byte, 400_000)); err == nil {
t.Fatal("a stream above the byte cap must be cut")
}
e.waitIdle()
if e.logs.Count(`"reason":"byte_cap"`) != 1 {
t.Fatalf("logs:\n%s", e.logs.String())
}
}
func TestLimits_StreamIdleAndMaxAge(t *testing.T) {
for _, tc := range []struct {
name string
mutate func(*Config)
reason string
}{
{"idle", func(c *Config) { c.StreamIdle = 300 * time.Millisecond }, "idle"},
{"max age", func(c *Config) { c.StreamMaxAge = 300 * time.Millisecond }, "max_age"},
} {
t.Run(tc.name, func(t *testing.T) {
release := make(chan struct{})
defer close(release)
e := newTestEnv(t, tc.mutate)
d := e.newDevice()
dc := e.connectOK(d)
dc.serve(holdHandler(d, release))
c, err := e.clientTLS(d)
if err != nil {
t.Fatal(err)
}
defer c.Close()
_ = c.SetReadDeadline(time.Now().Add(5 * time.Second))
if _, err := c.Read(make([]byte, 1)); err == nil {
t.Fatal("the stream should have been closed by the node")
}
e.waitIdle()
if e.logs.Count(`"reason":"`+tc.reason+`"`) != 1 {
t.Fatalf("logs:\n%s", e.logs.String())
}
})
}
}
func TestLimits_AttachTimeoutClosesTheClientAndForgetsTheStream(t *testing.T) {
e := newTestEnv(t, func(c *Config) { c.AttachTimeout = 300 * time.Millisecond })
d := e.newDevice()
dc := e.connectOK(d)
ds := dc.serve(nil)
ds.noAttach = true
if _, err := e.clientTLS(d); err == nil || !strings.Contains(err.Error(), "internal error") {
t.Fatalf("err = %v", err)
}
e.waitIdle()
e.srv.mu.Lock()
left := len(e.srv.pending)
e.srv.mu.Unlock()
if left != 0 {
t.Fatalf("%d pending streams leaked", left)
}
if e.logs.Count(`"reason":"attach_timeout"`) != 1 {
t.Fatalf("logs:\n%s", e.logs.String())
}
}
func TestShutdown_TellsDevicesAndClosesEverythingWithinGrace(t *testing.T) {
release := make(chan struct{})
defer close(release)
e := newTestEnv(t, nil)
d := e.newDevice()
dc := e.connectOK(d)
ds := dc.serve(holdHandler(d, release))
c, err := e.clientTLS(d)
if err != nil {
t.Fatal(err)
}
defer c.Close()
start := time.Now()
e.srv.Shutdown(500 * time.Millisecond)
if time.Since(start) > 4*time.Second {
t.Fatal("shutdown hung")
}
_ = c.SetReadDeadline(time.Now().Add(3 * time.Second))
if _, err := c.Read(make([]byte, 1)); err == nil {
t.Fatal("stream survived shutdown")
}
if _, err := net.DialTimeout("tcp", e.httpsAddr, 500*time.Millisecond); err == nil {
t.Fatal("listener still open after shutdown")
}
<-ds.done
if !containsLine(ds.otherLines(), "CLOSE shutdown") {
t.Fatalf("the device was not told: %q", ds.otherLines())
}
}
files/tunnel/stale_keys_grace_test.go 62 lines · 2.8 KB
package main
import (
"crypto/ed25519"
"crypto/rand"
"strings"
"testing"
"time"
)
// A paid device whose RENEW is refused only because the node cannot read the control plane's keys (they are older than the policy's window: the control plane is away) is
// a device the control plane did not answer: it gets the grace, as a device whose authorize call failed does. Before, the connection closed at valid_until (the chaos suite's
// real-device outage case): the node refused the session without recording that the control plane was down.
func TestOutage_ARenewRefusedBecauseTheKeysAreStaleStillEarnsThePaidGrace(t *testing.T) {
clk := newFakeClock()
// the node's own record of failed authorize calls must not be what gives the grace here (no authorize call is made: the session is refused before one): with it out of the way,
// only the refusal itself can say that the control plane is away
e, cp := httpNode(t, clk, func(c *Config) { c.DownQuiet = 1000 * time.Hour })
o := connectWithStream(t, e, devicePlan{validFor: 3 * time.Hour})
validUntil := o.validUntil()
cp.setDown(503)
e.pass(31*time.Minute, time.Minute) // the node's set of keys is now older than the window it may vouch for
_, newPriv, _ := ed25519.GenerateKey(rand.Reader)
const newKID = "cp-rotated-while-away"
o.conn.session = e.sessionSignedBy(newPriv, o.d, func(h, c map[string]any) { h["kid"] = newKID })
o.conn.send("RENEW " + o.conn.session)
if line := o.conn.recv(); line != "ERR control_unavailable" {
t.Fatalf("a RENEW with a key the node cannot vouch for: %q", line)
}
if !strings.Contains(e.logs.String(), `"reason":"unknown_kid"`) {
t.Fatal("the refusal was not the stale-keys one")
}
e.pass(validUntil.Sub(e.now())+time.Second, 30*time.Second)
if o.closed("CLOSE") || o.stream.dead(50*time.Millisecond) {
t.Fatal("a paid device was cut at valid_until although the control plane was away")
}
if !strings.Contains(e.logs.String(), `"msg":"control_grace"`) {
t.Fatal("no grace was given")
}
e.pass(policyDataGrace(), time.Minute)
eventually(t, "CLOSE expired once the grace is over", 3*time.Second, func() bool { return o.closed("CLOSE expired") })
}
func TestOutage_AHelloRefusedBecauseTheKeysAreStaleGivesNoGraceToAnythingBecauseThereIsNoConnectionYet(t *testing.T) {
clk := newFakeClock()
e, cp := httpNode(t, clk, nil)
cp.setDown(503)
e.pass(31*time.Minute, time.Minute)
d := e.newDevice()
_, newPriv, _ := ed25519.GenerateKey(rand.Reader)
line := e.connect(d).hello(func() string {
return e.sessionSignedBy(newPriv, d, func(h, c map[string]any) { h["kid"] = "cp-unknown" })
}, func(s, n string) string { return e.proof(d, s, n, nil) })
if line != "ERR control_unavailable" {
t.Fatalf("a HELLO on stale keys: %q", line)
}
if strings.Contains(e.logs.String(), `"msg":"control_grace"`) {
t.Fatal("a grace for a connection that never existed")
}
}
files/tunnel/statement_test.go 98 lines · 3.8 KB
package main
import (
"strings"
"testing"
"time"
)
// STATEMENT: the control plane's signed access statement, forwarded as it is, right after the OK, before any stream can be opened on the connection.
const testStatement = "eyJhbGciOiJFZERTQSIsInR5cCI6ImRldmljZS1hY2Nlc3MifQ.eyJzZXEiOjEyMzQ1fQ.c2lnbmF0dXJlLXNpZ25hdHVyZS1zaWduYXR1cmUtc2lnbmF0dXJl"
func statementPlan(e *testEnv, d *testDevice, access string) *devicePlan {
return &devicePlan{host: d.host, answer: func(int) AuthorizeResponse {
now := e.now()
return AuthorizeResponse{DeviceID: d.id, Label: d.label, Host: d.host, Status: statusActive, Plan: "paid", ValidUntil: now.Unix() + 3600, ServerTime: now.Unix(),
Policy: okPolicy{20, 45, 1800, 14400, 300}, Nodes: []any{}, Access: access}
}}
}
func TestStatement_IsForwardedUnchangedRightAfterTheOKOfAHelloAndOfARenew(t *testing.T) {
e := newTestEnv(t, nil)
d := e.newDeviceNoPlan()
e.cp.set(d.id, statementPlan(e, d, testStatement))
dc := e.connect(d)
dc.session = e.session(d, nil)
dc.send("HELLO 2 " + dc.session)
m, _ := parseLine(dc.recv(), dirN2D)
dc.send("PROOF " + e.proof(d, dc.session, m.str("nonce"), nil))
if line := dc.recv(); !strings.HasPrefix(line, "OK ") {
t.Fatalf("%q", line)
}
if line := dc.recv(); line != "STATEMENT "+testStatement {
t.Fatalf("the line after the OK: %q", line)
}
// a client stream opened now reaches a device that has the statement already: the OPEN comes after it
ds := dc.serve(nil)
c, err := e.clientTLS(d)
if err != nil {
t.Fatal(err)
}
_ = c.Close()
if ds.openCount() != 1 {
t.Fatalf("opens %d", ds.openCount())
}
// and at every RENEW, with the control plane's newer statement
const newer = "eyJhbGciOiJFZERTQSJ9.eyJzZXEiOjEyMzQ2fQ.c2lnbmF0dXJl"
e.cp.set(d.id, statementPlan(e, d, newer))
if r := dc.renew(); !strings.HasPrefix(r, "OK ") {
t.Fatalf("%q", r)
}
eventually(t, "the newer statement", 2*time.Second, func() bool { return containsLine(ds.otherLines(), "STATEMENT "+newer) })
e.checkTranscript()
}
func TestStatement_NoStatementNoLineAndABadOneIsDroppedNeverRelayedNeverAPanic(t *testing.T) {
e := newTestEnv(t, nil)
for name, access := range map[string]string{
"none": "",
"not a jws": "this is not a token",
"two segments": "aaaa.bbbb",
"too long": strings.Repeat("a", policyData.Protocol.StatementLineMaxBytes) + ".b.c",
"a space in it": "aaaa.bb bb.cccc",
} {
d := e.newDeviceNoPlan()
e.cp.set(d.id, statementPlan(e, d, access))
dc := e.connectOK(d)
ds := dc.serve(nil)
time.Sleep(60 * time.Millisecond)
for _, line := range ds.otherLines() {
if strings.HasPrefix(line, "STATEMENT") {
t.Errorf("%s: forwarded %.40q", name, line)
}
}
if !e.srv.Online(d.host) {
t.Errorf("%s: the connection did not come up", name)
}
}
// the longest line the grammar allows is forwarded
d := e.newDeviceNoPlan()
longest := strings.Repeat("a", policyData.Protocol.StatementLineMaxBytes-len("STATEMENT ")-1-4) + ".b.c"
if statementLine(longest) == "" {
t.Fatal("the longest statement the grammar allows was refused")
}
e.cp.set(d.id, statementPlan(e, d, longest))
dc := e.connectOK(d)
ds := dc.serve(nil)
eventually(t, "the longest statement", 3*time.Second, func() bool { return containsLine(ds.otherLines(), "STATEMENT "+longest) })
}
func TestStatement_AnAccessTheControlPlaneSendsThatTheGrammarRefusesIsABadAnswerOverHTTP(t *testing.T) {
bad := `{"ok":true,"v":1,"device_id":"` + randomHex(16) + `","label":"` + label20() + `","host":"x","status":"active","plan":"paid","valid_until":1793000000,"server_time":1793000000,` +
`"policy":{"ping_interval_s":20,"dead_peer_s":45,"stream_idle_s":1800,"stream_max_age_s":14400,"renew_before_valid_until_s":300},"nodes":[],"access":"not a jws"}`
if _, err := parseAuthorizeResponse([]byte(bad)); err == nil {
t.Fatal("an answer with a statement that is not a JWS was accepted")
}
}
files/tunnel/storm_test.go 107 lines · 3.3 KB
package main
import (
"bufio"
"fmt"
"io"
"strings"
"sync"
"sync/atomic"
"syscall"
"testing"
"time"
)
// A node that restarts meets every device at once. The control plane is asked by at most authorize_max_concurrent calls at a time, what waits is
// bounded, the rest is told to try again, and the devices (who back off) all get in.
func TestStorm_AFleetReconnectsAtOnceAfterANodeRestartAndTheControlPlaneNeverSeesMoreThanTheBound(t *testing.T) {
var lim syscall.Rlimit
_ = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &lim)
devices := 1000
if lim.Cur < 8192 {
devices = max(60, int(lim.Cur)/12)
}
e, cp := httpNode(t, nil, func(c *Config) {
c.HelloTimeout, c.FirstLineTimeout, c.AuthorizeTimeout = 10*time.Second, 20*time.Second, 10*time.Second
c.MaxUnauth = 64 // small, so that the storm really exceeds what may wait for the control plane
})
cp.mu.Lock()
cp.delay = 25 * time.Millisecond
cp.mu.Unlock()
all := make([]*testDevice, devices)
for i := range all {
all[i] = e.newDevice()
}
var connected, refusedBusy atomic.Int64
var wg sync.WaitGroup
for _, d := range all {
wg.Add(1)
go func(d *testDevice) {
defer wg.Done()
for attempt := 0; attempt < 60; attempt++ {
line := helloOnce(e, d)
switch {
case strings.HasPrefix(line, "OK "):
connected.Add(1)
return
case line == "ERR control_unavailable" || strings.HasPrefix(line, "unreachable"):
refusedBusy.Add(1)
time.Sleep(time.Duration(30+attempt*10) * time.Millisecond) // the device's backoff
default:
t.Errorf("a device was refused with %q", line)
return
}
}
t.Errorf("a device never got in")
}(d)
}
wg.Wait()
if int(connected.Load()) != devices {
t.Fatalf("%d of %d devices connected", connected.Load(), devices)
}
if peak := cp.maxInflight(); peak > policyData.Authorization.AuthorizeMaxConcur {
t.Fatalf("the control plane saw %d authorizations at once, the bound is %d", peak, policyData.Authorization.AuthorizeMaxConcur)
}
if refusedBusy.Load() == 0 {
t.Log("(the storm never exceeded what may wait; the bound was still respected)")
}
for _, d := range all[:5] {
if !e.srv.Online(d.host) {
t.Fatal("a connected device is not online")
}
}
t.Log(fmt.Sprintf("%d devices, %d told to try again, peak %d at the control plane", devices, refusedBusy.Load(), cp.maxInflight()))
}
// helloOnce is a whole HELLO of d on its own connection, as a device does it: the node's final line, or "unreachable: ..." when the node would not
// even take the connection (it is full of connections that have not authenticated yet). The connection is left open: a device that got in stays in.
func helloOnce(e *testEnv, d *testDevice) string {
c, err := e.dialNode()
if err != nil {
return "unreachable: " + err.Error()
}
br := bufio.NewReader(c)
read := func() string {
_ = c.SetReadDeadline(time.Now().Add(15 * time.Second))
l, err := br.ReadString('\n')
if err != nil {
return "unreachable: " + err.Error()
}
return strings.TrimSuffix(l, "\n")
}
session := e.session(d, nil)
_, _ = io.WriteString(c, "HELLO 2 "+session+"\n")
line := read()
if nonce, ok := strings.CutPrefix(line, "CHALLENGE "); ok {
_, _ = io.WriteString(c, "PROOF "+e.proof(d, session, nonce, nil)+"\n")
line = read()
}
if !strings.HasPrefix(line, "OK ") {
_ = c.Close()
} else {
e.t.Cleanup(func() { _ = c.Close() })
}
return line
}
files/tunnel/unauth_reserve_test.go 153 lines · 6.0 KB
package main
import (
"bufio"
"context"
"crypto/tls"
"net"
"net/netip"
"testing"
"time"
)
// The unauthenticated budget keeps a reserve (policy node_limits.openai_reserved_pct), as the stream budget does. Before it, every connection took a slot of the whole
// budget before its ClientHello, so twenty addresses holding the most one source may hold (16 half-open connections each) took all 256 of them and every other connection,
// a ChatGPT client or a device reconnecting, was dropped with no answer for as long as the attack lasted (the chaos suite's slowloris check).
func TestNode_TwentyAnonymousSourcesCannotStarveChatGPTNorADeviceThatReconnects(t *testing.T) {
release := make(chan struct{})
e := newVIPEnv(t, func(e *testEnv, c *Config) {
c.PerIPConcurrent, c.MaxUnauth, c.OpenAIReservedPct, c.HelloTimeout = 4, 16, 25, 20*time.Second
c.PrefixFetch = func(context.Context) ([]byte, error) { return []byte(listOf("192.0.2.0/24")), nil }
})
eventually(t, "the OpenAI list", 3*time.Second, func() bool { return e.srv.openai.size() == 1 })
d := e.newDevice()
dc := e.connectOK(d)
dc.serve(holdHandler(d, release))
defer close(release)
// a second device, at an address of its own, connects once and lets go: it will be a device that reconnects
const home = "198.51.100.200"
back := e.newDevice()
first := e.connectOKFrom(home, back)
_ = first.c.Close()
eventually(t, "the node saw the control connection go", 3*time.Second, func() bool {
return e.srv.ips.knownDevice(netip.MustParseAddr(home)) && !e.srv.ips.isDevice(netip.MustParseAddr(home))
})
// twenty anonymous sources, each holding the most it may hold, saying nothing
var idle []net.Conn
for source := 0; source < 20; source++ {
ip := "198.51.100." + itoa(100+source)
for i := 0; i < 4; i++ {
c, err := e.vip.dial(ip, e.httpsAddr)
if err != nil {
t.Fatal(err)
}
defer c.Close()
idle = append(idle, c)
}
}
share := e.srv.anonUnauthMax()
if share != 12 {
t.Fatalf("the anonymous share is %d, want 16 less the reserve of 25 percent: 12", share)
}
eventually(t, "the anonymous share is full and the rest were refused", 4*time.Second, func() bool {
return e.srv.anonUnauth.Load() == share && e.rejected("unauth_anon_limit") >= int64(len(idle))-share
})
if got := e.srv.unauth.Load(); got > share {
t.Fatalf("the attack holds %d unauthenticated slots of 16: more than its share of %d", got, share)
}
// ChatGPT (an address of the OpenAI list) is served through the attack
good, err := e.clientFrom("192.0.2.9", d)
if err != nil {
t.Fatalf("a client from the OpenAI list was turned away while twenty anonymous sources held the budget: %v", err)
}
_ = good.Close()
// a device that reconnects (it held a control connection a moment ago, from this address) gets its HELLO through; one nobody knows waits
_ = e.connectOKFrom(home, back)
stranger := e.newDevice()
if conn, err := e.dialFrom("198.51.100.201"); err == nil {
line := (&devConn{t: t, e: e, d: stranger, c: conn, br: bufio.NewReaderSize(conn, 1<<17)})
if _, err := line.try(2 * time.Second); err == nil {
t.Fatalf("a device from an address nobody knows got an answer through a full anonymous share")
}
}
// an anonymous source with no reason to be trusted is the one that waits: the share is the price of the reserve
if _, err := e.clientFrom("203.0.113.77", d); err == nil {
t.Fatalf("an anonymous source got through a full anonymous share")
}
// when the attack ends the share is free again
for _, c := range idle {
_ = c.Close()
}
eventually(t, "the anonymous share is free", 4*time.Second, func() bool { return e.srv.anonUnauth.Load() == 0 })
later, err := e.clientFrom("203.0.113.77", d)
if err != nil {
t.Fatalf("an anonymous source is still refused after the attack: %v", err)
}
_ = later.Close()
}
// dialFrom connects to the node's own name the way a device at ip does (the source address is the test's choice).
func (e *testEnv) dialFrom(ip string) (*tls.Conn, error) {
raw, err := e.vip.dial(ip, e.httpsAddr)
if err != nil {
return nil, err
}
c := tls.Client(raw, &tls.Config{ServerName: testNodeHost, RootCAs: e.nodePool, NextProtos: []string{policyData.Protocol.ALPN}, MinVersion: tls.VersionTLS13})
_ = c.SetDeadline(time.Now().Add(3 * time.Second))
if err := c.Handshake(); err != nil {
_ = raw.Close()
return nil, err
}
_ = c.SetDeadline(time.Time{})
e.t.Cleanup(func() { _ = c.Close() })
return c, nil
}
// connectOKFrom is connectOK from a source address of the test's choice.
func (e *testEnv) connectOKFrom(ip string, d *testDevice) *devConn {
e.t.Helper()
c, err := e.dialFrom(ip)
if err != nil {
e.t.Fatalf("the device at %s was turned away: %v", ip, err)
}
dc := &devConn{t: e.t, e: e, d: d, c: c, br: bufio.NewReaderSize(c, 1<<17)}
line := dc.hello(func() string { return e.session(d, nil) }, func(s, n string) string { return e.proof(d, s, n, nil) })
m, err := parseLine(line, dirN2D)
if err != nil || m.str("verb") != "OK" {
e.t.Fatalf("handshake for %s from %s: %q", d.id, ip, line)
}
dc.ok = m
return dc
}
func TestIPLimits_ADeviceThatLetGoIsKnownForHintTTLAndThenForgotten(t *testing.T) {
l, clk := newTestLimits(t, func(c *Config) { c.HintTTL = 15 * time.Minute })
addr := netip.MustParseAddr("203.0.113.5")
if l.knownDevice(addr) {
t.Fatal("an address nobody has seen is known as a device's")
}
unmark := l.markDevice(addr)
if !l.knownDevice(addr) {
t.Fatal("an address that holds a control connection is not known as a device's")
}
unmark()
if !l.knownDevice(addr) {
t.Fatal("a device that just let go is an anonymous source again at once: its reconnection would wait for the anonymous share")
}
clk.Advance(14 * time.Minute)
if !l.knownDevice(addr) {
t.Fatal("known for less than hint_ttl_s after the last connection ended")
}
clk.Advance(2 * time.Minute)
if l.knownDevice(addr) {
t.Fatal("known for longer than hint_ttl_s")
}
l.markDevice(addr)()
clk.Advance(20 * time.Minute)
l.Sweep()
if len(l.past) != 0 {
t.Fatalf("the sweep left %d remembered devices", len(l.past))
}
}