diff --git a/docs/changelog-20260807-dc0-tailscale-install.md b/docs/changelog-20260807-dc0-tailscale-install.md index b9246d1..f76e8d6 100644 --- a/docs/changelog-20260807-dc0-tailscale-install.md +++ b/docs/changelog-20260807-dc0-tailscale-install.md @@ -64,3 +64,44 @@ (naming convention owed; cosmetic, does not affect the join -- tailnet name set by `--hostname`). - F2: raw `grep DOCFIX-` returns a stray `DOCFIX-1004` string somewhere (ledger-scan's next-free is 213); verify it is not a real assignment collision. + +## UPDATE (same session) -- install incident + fix + live re-join + +**INCIDENT:** the first live join succeeded (node registered, `machineAuthorized=true`) but bringing +`.7` up as a subnet router with `--accept-routes` **blackholed its own L3 on both legs** and locked +us out (L2/ARP alive, no IP). Root cause: a subnet router that also advertises the `/22` its own +SSH path lives in must NOT `--accept-routes` -- it imports a route that sends its own return traffic +into `tailscale0`. office1 tolerates accept-routes only because it is steady-state / reached +differently. Reboot-race recovery lost (tailscaled reconnects in seconds with a cached control key). +**Recovered via `qemu-nbd` offline mask of `tailscaled`** on the rack (VM disk +`/var/lib/libvirt/vr1-dc0-inner/vr1-dc0-tailscale-01-disk.qcow2`, root = `nbd0p2`), then purged +tailscale + wiped state for a clean base (operator: "remove and rebuild tailscale"). + +**SECONDARY:** the pre-auth key leaked -- it was on the `tailscale up` **argv** (visible in `ps`) and +reached a session transcript. Operator reused the existing key for the re-join (accepted); it should +still be rotated. **The tool now passes `--authkey=file:` so the value never touches argv again.** + +**TOOL FIXES (this commit, harness 27/0, gauntlet ALL GREEN 102, repo-lint 0-fail):** +- `install`: **dropped `--accept-routes`** (advertise-only subnet router) -- the incident fix; and + `--authkey=file:` (no argv leak; `$TS_AUTHKEY` env is staged to a 0600 temp file). +- `check`: two new assertions with failing-direction fixtures -- **own-subnet-not-via-tailscale0** + (the blackhole guard) and **control-plane reachability** (HTTP response from `$LOGIN_SERVER`, not + just DNS -- the green-except-route failure class). +- Revert: `git revert` this commit; the prior form is the accept-routes/argv-key version. + +**LIVE RE-JOIN (advertise-only, safety-net protected):** disabled v6 at runtime on `.7` (fast join; +no global v6 to lose), armed a 180s `systemd-run` auto-`tailscale down` net, ran `install` -> **up +Running/Online, TSIP `100.64.0.57`, own-subnet NOT hijacked, control reachable, .7 stayed +reachable** (no lockout). `check` FAILs only on the route pending Headscale approval (expected). +Safety net cancelled, key removed from `.7`. + +**OWED / follow-ups:** +- **Operator: approve `10.12.8.0/22` on `vr1-dc0-tailscale` (100.64.0.57)** in Headscale; delete the + **stale node `100.64.0.56`** from the failed first attempt. Then `check` -> green + Horizon confirm. +- `disable_ipv6` on `.7` is **runtime-only** (not persisted) -- on reboot v6 returns and the re-join + takes ~7 min (v6 control backoff before v4 fallback). Decide whether to persist it (fast reconnect) + vs accept the delay. NOT baked into `prep` (unproven-persistence; advisor guidance). +- **F3 (new): the maas snap refresh (rev 41649) wiped the `vr1-dc0-region` CLI profile** on voffice1 + -- a true MAAS rebuild is blocked until the profile is re-logged in (region API key + tunnel to + `10.12.8.6:5240`). Likely affects dc1 too. Snap-per-revision profile store, the known trap. +- Rotate the leaked pre-auth key once dc0 is confirmed green. diff --git a/scripts/site-tailscale.sh b/scripts/site-tailscale.sh index 85c5340..8b4a695 100755 --- a/scripts/site-tailscale.sh +++ b/scripts/site-tailscale.sh @@ -66,6 +66,7 @@ TS="${TAILSCALE:-tailscale}" # overridable so the harness injects a fake SYSCTL="${SYSCTL:-sysctl}" # overridable so the harness fakes forwarding SYSCTL_D="${TS_SYSCTL_D:-/etc/sysctl.d}" # overridable so the harness writes to a tmp dir +CURL="${CURL:-curl}" # overridable so the harness fakes control reach say(){ printf '%s\n' "$*"; } # jget [...] -- navigate nested keys of `tailscale status --json`; no eval (a # bracket-key path in an eval string collides its own quotes). Prints Python repr (lists incl.). @@ -123,18 +124,27 @@ install(){ command -v "$TS" >/dev/null 2>&1 || { say "install: tailscale not present -- run 'prep $SITE' first"; return 4; } - local key="${TS_AUTHKEY:-}" - [ -z "$key" ] && [ -n "${TS_AUTHKEY_FILE:-}" ] && [ -s "${TS_AUTHKEY_FILE:-}" ] && key="$(cat "$TS_AUTHKEY_FILE")" - [ -n "$key" ] || { say "install: no pre-auth key (\$TS_AUTHKEY / \$TS_AUTHKEY_FILE)"; return 4; } - # SNAT default (on) per ruling (d): no --snat-subnet-routes flag. --accept-routes mirrors - # office1's RouteAll=True. --advertise-tags ONLY when TS_TAG is set (untagged VR1 by default). + # Resolve the key to a FILE and hand it to tailscale via the file: scheme so the secret never + # lands on argv / in ps (2026-08-07: an --authkey= form leaked the key into ps AND a + # session transcript). If only $TS_AUTHKEY is given, stage it to a 0600 temp file, never argv. + local keyfile="${TS_AUTHKEY_FILE:-}" tmpkey="" + if [ -z "$keyfile" ] && [ -n "${TS_AUTHKEY:-}" ]; then + tmpkey="$(mktemp)"; chmod 600 "$tmpkey"; printf '%s' "$TS_AUTHKEY" > "$tmpkey"; keyfile="$tmpkey" + fi + { [ -n "$keyfile" ] && [ -s "$keyfile" ]; } || { say "install: no pre-auth key (\$TS_AUTHKEY_FILE / \$TS_AUTHKEY)"; return 4; } local tagflag=() [ -n "$TAG" ] && tagflag=(--advertise-tags="$TAG") - "$TS" up --login-server="$LOGIN_SERVER" --authkey="$key" \ - --advertise-routes="$METAL_ADMIN_CIDR" --accept-routes "${tagflag[@]}" \ - --hostname="${SITE}-tailscale" >/dev/null 2>&1 \ - || { say "install: 'tailscale up' failed (key rejected, tag not authorised, or control-plane unreachable)"; return 4; } - say "install: tailscale up issued for $SITE advertising $METAL_ADMIN_CIDR${TAG:+ as $TAG}" + # Subnet router = ADVERTISE ONLY. NO --accept-routes: on a DC .7 whose SSH path lives INSIDE the + # advertised /22, consuming routes blackholed the node's own L3 and locked us out (2026-08-07 + # incident). A subnet router advertises; it does not need to consume routes. SNAT default on + # (ruling d, no flag). --advertise-tags only when TS_TAG set (untagged VR1 by default). + local rc=0 + "$TS" up --login-server="$LOGIN_SERVER" --authkey="file:$keyfile" \ + --advertise-routes="$METAL_ADMIN_CIDR" "${tagflag[@]}" \ + --hostname="${SITE}-tailscale" >/dev/null 2>&1 || rc=$? + [ -n "$tmpkey" ] && rm -f "$tmpkey" + [ "$rc" -eq 0 ] || { say "install: 'tailscale up' failed (key rejected, tag not authorised, or control-plane unreachable)"; return 4; } + say "install: tailscale up issued for $SITE advertising $METAL_ADMIN_CIDR${TAG:+ as $TAG} (advertise-only)" say "NOTE: Headscale must APPROVE the route (manual, or autoApprovers if tagged) -- run 'check' to confirm." } @@ -158,6 +168,18 @@ fi if own_leg_matches; then say "OK: router holds a .7 metal-admin leg in $METAL_ADMIN_CIDR" else say "FAIL: no .7 metal-admin leg in $METAL_ADMIN_CIDR on this host -- wrong VM or unconfigured"; rc=1; fi + # Own-subnet guard: the node must NOT route its OWN advertised /22 via tailscale0 -- doing so + # blackholes its metal-admin path and locks the operator out (2026-08-07 incident, the reason + # --accept-routes is gone). This is a green-except-route failure the other checks cannot see. + if ip route show "$METAL_ADMIN_CIDR" 2>/dev/null | grep -q tailscale0; then + say "FAIL: $METAL_ADMIN_CIDR routed via tailscale0 -- node is blackholing its own subnet (2026-08-07)"; rc=1 + else say "OK: own subnet $METAL_ADMIN_CIDR not routed via tailscale0"; fi + # Control-plane reachability: a subnet router that cannot reach Headscale registers/stays down + # while every OTHER check but the route can read green (advisor 2026-08-07). Assert a live HTTP + # response from the control URL, not just DNS. + local code; code="$("$CURL" -s -o /dev/null -m 8 -w '%{http_code}' "$LOGIN_SERVER" 2>/dev/null)" + if [ -n "$code" ] && [ "$code" != 000 ]; then say "OK: control plane reachable ($LOGIN_SERVER -> HTTP $code)" + else say "FAIL: control plane $LOGIN_SERVER unreachable (http='$code') -- node cannot register/stay up"; rc=1; fi return $rc } diff --git a/tests/site-tailscale/run-tests.sh b/tests/site-tailscale/run-tests.sh index 2d328a5..82d3eb9 100755 --- a/tests/site-tailscale/run-tests.sh +++ b/tests/site-tailscale/run-tests.sh @@ -1,17 +1,19 @@ #!/usr/bin/env bash # tests/site-tailscale/run-tests.sh -- offline harness for scripts/site-tailscale.sh. -# Fakes `tailscale` (status --json / up / version), `ip`, `sysctl` (forwarding), `dpkg` (prep -# install); real python3 for jget. NO network, mutates nothing outside mktemp. Proves each exit -# path can FIRE (a new assertion is decoration until it has a failing-direction fixture): +# Fakes tailscale (status/up/version), ip (addr + route), sysctl (forwarding), dpkg (prep), +# curl (control reach); real python3 for jget. NO network, mutates nothing outside mktemp. +# Proves each exit path can FIRE (a new assertion is decoration until it has a failing fixture): # prep-happy -> 0 (pkg present + forwarding on) # prep-fromdeb -> 0 (pkg absent, $TS_DEB staged, dpkg installs it, forwarding on) -# prep-noforward -> 4 (forwarding sysctl reads 0 -- the load-bearing subnet-router property) +# prep-noforward -> 4 (forwarding sysctl reads 0 -- load-bearing subnet-router property) # prep-nopkg-nodeb-> 4 (pkg absent and no $TS_DEB staged) -# check-happy -> 0 (Running + route approved + forwarding + untagged + .7 leg) +# check-happy -> 0 (Running + route approved + forwarding + untagged + leg + no-hijack + control-up) # check-notrun -> 1 (BackendState != Running) # check-noroute -> 1 (metal-admin CIDR not in approved PrimaryRoutes) -# check-noforward -> 1 (IP forwarding off -- green everywhere else, forwards nothing) +# check-noforward -> 1 (IP forwarding off) # check-noleg -> 1 (no .7 metal-admin leg on the host) +# check-hijack -> 1 (own /22 routed via tailscale0 -- the 2026-08-07 blackhole) +# check-ctrldown -> 1 (control plane unreachable -- registers/stays down, green-except-route) # check-tag-happy -> 0 (TS_TAG set + node tagged) # check-tag-notag -> 1 (TS_TAG set but node NOT tagged) # install-happy -> 0 (untagged up ok + check green) @@ -19,9 +21,8 @@ # install-nokey -> 4 (no pre-auth key) # install-noTS -> 4 (tailscale binary absent -> "run prep first") # install-upfail -> 4 (tailscale up rejected) -# badmode/badsite/noargs -> 2 -# dc1 resolves its own CIDR (10.12.68.0/22), not dc0's -# site-table rows carry MEASURED tags (hard rule 2) +# key-not-in-argv (install passes --authkey=file:, the key VALUE never reaches argv) +# badmode/badsite/noargs -> 2 ; dc1 resolves its own CIDR ; site rows carry MEASURED tags # Exit: 0 all pass | 1 any case failed. ASCII + LF. set -uo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -31,41 +32,41 @@ PASS=0; FAIL=0 BIN="$TMP/bin"; mkdir -p "$BIN" "$TMP/sysctl.d" printf 'fake-deb-content\n' > "$TMP/fake.deb" +SECRET='tskey-SECRET-do-not-leak-0xdeadbeef'; printf '%s' "$SECRET" > "$TMP/authkey.file" -# ---- fake tailscale -------------------------------------------------------- +# ---- fake tailscale (records `up` argv so we can prove the key never lands there) ---------- cat > "$BIN/tailscale" <<'FB' #!/usr/bin/env bash mode="${TS_FAKE_MODE:-happy}"; cidr="${TS_FAKE_CIDR:-10.12.8.0/22}" case "$1" in version) echo "1.98.9"; exit 0;; - up) [ "$mode" = upfail ] && exit 1; exit 0;; + up) [ -n "${TS_ARGV_LOG:-}" ] && printf '%s\n' "$*" >> "$TS_ARGV_LOG"; [ "$mode" = upfail ] && exit 1; exit 0;; status) - # $2 == --json st='"Running"'; routes="[\"$cidr\"]" tags='[]'; [ -n "${TS_TAG:-}" ] && tags="[\"$TS_TAG\"]" - case "$mode" in - notrun) st='"Stopped"';; - noroute) routes='[]';; - notag) tags='[]';; - esac - printf '{"BackendState":%s,"Self":{"PrimaryRoutes":%s,"Tags":%s}}\n' "$st" "$routes" "$tags" - exit 0;; + case "$mode" in notrun) st='"Stopped"';; noroute) routes='[]';; notag) tags='[]';; esac + printf '{"BackendState":%s,"Self":{"PrimaryRoutes":%s,"Tags":%s}}\n' "$st" "$routes" "$tags"; exit 0;; *) exit 0;; esac FB chmod +x "$BIN/tailscale" -# ---- fake ip (own_leg_matches) -------------------------------------------- +# ---- fake ip: addr show (own_leg_matches) + route show (own-subnet guard) ------------------ cat > "$BIN/ip" <<'FB' #!/usr/bin/env bash -# emit a .7 metal-admin leg unless TS_FAKE_MODE=noleg +case "$*" in + *"route show"*) # own-subnet guard: emit a tailscale0 route only when hijack fixture set + [ "${TS_FAKE_HIJACK:-0}" = 1 ] && echo "${TS_FAKE_CIDR:-10.12.8.0/22} dev tailscale0 scope link" + exit 0;; +esac +# addr show path [ "${TS_FAKE_MODE:-}" = noleg ] && { echo "1: lo inet 127.0.0.1/8 scope host lo"; exit 0; } echo "2: enp1s0 inet ${TS_FAKE_LEG:-10.12.8.7}/22 scope global enp1s0" exit 0 FB chmod +x "$BIN/ip" -# ---- fake sysctl (forwarding) --------------------------------------------- +# ---- fake sysctl (forwarding) ---------------------------------------------- cat > "$BIN/sysctl" <<'FB' #!/usr/bin/env bash fwd="${TS_FAKE_FWD:-1}" @@ -78,8 +79,15 @@ FB chmod +x "$BIN/sysctl" +# ---- fake curl (control-plane reachability) -------------------------------- +cat > "$BIN/curl" <<'FB' +#!/usr/bin/env bash +[ "${TS_FAKE_CTRL:-up}" = down ] && { printf '000'; exit 7; } +printf '200'; exit 0 +FB +chmod +x "$BIN/curl" + # ---- fake dpkg (prep install-from-deb) ------------------------------------ -# "installs" by copying the fake tailscale to $TS_INSTALL_TARGET so command -v succeeds after. cat > "$BIN/dpkg" <&1)"; code=$? if [ "$code" -eq "$exp" ]; then PASS=$((PASS+1)); echo "PASS: $name (exit $code)" @@ -106,39 +116,46 @@ } echo "=== site-tailscale harness ===" -# prep cases +# prep run prep-happy 0 happy prep vr1-dc0 CASE_TS="$BIN/staged-ts" CASE_DEB="$TMP/fake.deb" run prep-fromdeb 0 happy prep vr1-dc0 CASE_FWD=0 run prep-noforward 4 happy prep vr1-dc0 CASE_TS="$BIN/nope" run prep-nopkg-nodeb 4 happy prep vr1-dc0 -# check cases (default untagged) +# check (default untagged) run check-happy 0 happy check vr1-dc0 run check-notrun 1 notrun check vr1-dc0 run check-noroute 1 noroute check vr1-dc0 -CASE_FWD=0 run check-noforward 1 happy check vr1-dc0 +CASE_FWD=0 run check-noforward 1 happy check vr1-dc0 run check-noleg 1 noleg check vr1-dc0 -# tagged mode (opt-in via TS_TAG) +CASE_HIJACK=1 run check-hijack 1 happy check vr1-dc0 +CASE_CTRL=down run check-ctrldown 1 happy check vr1-dc0 +# tagged mode CASE_TAG="tag:subnet-router" run check-tag-happy 0 happy check vr1-dc0 CASE_TAG="tag:subnet-router" run check-tag-notag 1 notag check vr1-dc0 -# install cases +# install CASE_AUTHKEY="tskey-fake" run install-happy 0 happy install vr1-dc0 CASE_TAG="tag:subnet-router" CASE_AUTHKEY="tskey-fake" run install-tag-happy 0 happy install vr1-dc0 run install-nokey 4 happy install vr1-dc0 CASE_AUTHKEY="tskey-fake" run install-upfail 4 upfail install vr1-dc0 CASE_TS="$BIN/nope" CASE_AUTHKEY="tskey-fake" run install-noTS 4 happy install vr1-dc0 + +# key-not-in-argv: install via TS_AUTHKEY_FILE must pass --authkey=file:, never the value +CASE_KEYFILE="$TMP/authkey.file" CASE_ARGV_LOG="$TMP/argv.log" run install-keyfile 0 happy install vr1-dc0 +if grep -q 'authkey=file:' "$TMP/argv.log" 2>/dev/null && ! grep -qF "$SECRET" "$TMP/argv.log" 2>/dev/null; then + PASS=$((PASS+1)); echo "PASS: key-not-in-argv (file: scheme used, value absent from argv)" +else FAIL=$((FAIL+1)); echo "FAIL: key-not-in-argv"; sed 's/^/ /' "$TMP/argv.log" 2>/dev/null; fi + # arg/site validation run badmode 2 happy frobnicate vr1-dc0 run badsite 2 happy check vr1-dc9 run noargs 2 happy - -# dc1 resolves its OWN cidr (not dc0's) -- run check-happy with dc1 fixture +# dc1 resolves its OWN cidr CASE_CIDR='10.12.68.0/22' CASE_LEG='10.12.68.7' run dc1-cidr 0 happy check vr1-dc1 -out="$(PATH="$BIN:$PATH" TAILSCALE="$BIN/tailscale" SYSCTL="$BIN/sysctl" TS_SYSCTL_D="$TMP/sysctl.d" \ +out="$(PATH="$BIN:$PATH" TAILSCALE="$BIN/tailscale" SYSCTL="$BIN/sysctl" CURL="$BIN/curl" TS_SYSCTL_D="$TMP/sysctl.d" \ TS_FAKE_MODE=happy TS_FAKE_CIDR='10.12.68.0/22' TS_FAKE_LEG='10.12.68.7' \ bash "$SUT" check vr1-dc1 2>&1)" if printf '%s' "$out" | grep -q '10.12.68.0/22'; then PASS=$((PASS+1)); echo "PASS: dc1 emits 10.12.68.0/22" else FAIL=$((FAIL+1)); echo "FAIL: dc1 site cidr"; echo "$out" | sed 's/^/ /'; fi - # site-table rows must carry MEASURED tags (hard rule 2) for s in 'vr1-dc0.*MEASURED' 'vr1-dc1.*MEASURED'; do if grep -qE "$s" "$SUT"; then PASS=$((PASS+1)); echo "PASS: site row tagged MEASURED ($s)"