diff --git a/.claude/settings.json b/.claude/settings.json index d89dbbd..c8b3df6 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -1,4 +1,10 @@ { + "env": { + "CAVEMAN_DEFAULT_MODE": "off" + }, + "enabledPlugins": { + "caveman@caveman": false + }, "permissions": { "allow": [ "Bash(git status*)", "Bash(git diff*)", "Bash(git log*)", "Bash(git pull*)", @@ -16,9 +22,22 @@ "Bash(bash scripts/preflight.sh*)", "Bash(python3 scripts/repo_lint.py*)", "Bash(python3 scripts/provider-bundle-check.py*)", + "Bash(python3 scripts/dc-dc-whole-host-budget.py*)", "Bash(bash tests/*)", "Bash(OPNSENSE_API_HOST=* bash scripts/opnsense-api.sh GET *)", - "Bash(bash scripts/opnsense-plugins.sh list*)" + "Bash(bash scripts/opnsense-plugins.sh list*)", + "Bash(bash scripts/creds-audit.sh *)", + "Bash(bash scripts/dc-dc-ceph-disk-budget.sh *)", + "Bash(bash scripts/site-baseleg.sh check*)", + "Bash(tofu validate*)", + "Bash(tofu -chdir=* validate*)", + "Bash(tofu state list*)", + "Bash(tofu -chdir=* state list*)", + "Bash(tofu state show*)", + "Bash(tofu providers schema*)", + "Bash(tofu version*)", + "Bash(ip route*)", + "Bash(systemctl is-active *)" ], "ask": [ "Bash(juju destroy-model *)", "Bash(juju remove-machine *)", @@ -42,9 +61,11 @@ "Bash(git rebase*)", "Bash(git branch -D *)", "Bash(bash scripts/* apply*)", "Bash(bash scripts/* install*)", "Bash(ssh * sudo *)", + "Bash(ssh *'sudo *)", "Bash(ssh *\"sudo *)", "Bash(ssh *sudo -n *)", + "Bash(*creds-matrix.py *--privileged*)", "Bash(scp *)", "Bash(* | ssh *)", - "Bash(ssh * virsh *)", + "Bash(ssh * virsh *)", "Bash(ssh *'virsh *)", "Bash(ssh *\"virsh *)", "Skill(*)", "Edit(.claude/settings.json)", "Edit(.claude/settings.local.json)" ], @@ -54,6 +75,7 @@ "Bash(juju destroy-controller *)", "Bash(maas list*)", "Bash(git push --force*)", "Bash(git push -f*)", + "Bash(*python3 -m scripts*)", "Bash(*caveman-compress*)", "Read(~/vault-init/**)", "Read(~/as-executed/**)", "Read(~/tenant-*/**)", "Read(**/*-cred.txt)", "Read(**/*appcred*)", "Edit(~/vault-init/**)" diff --git a/.claude/skills/openstack-cloud-ops-consolidated-20260727.md b/.claude/skills/openstack-cloud-ops-consolidated-20260727.md new file mode 100644 index 0000000..f0065e6 --- /dev/null +++ b/.claude/skills/openstack-cloud-ops-consolidated-20260727.md @@ -0,0 +1,1589 @@ + + +# openstack-cloud-ops -- consolidated skill (Chat upload snapshot, 2026-07-27) + +================================================================================ +## SKILL.md (main entry -- always in context) +================================================================================ + +--- +name: openstack-cloud-ops +description: "Operate, install, extend, and troubleshoot the Omega Cloud - a commercial multi-tenant Charmed OpenStack (Caracal 2024.1) deployment managed with Juju and MAAS, with Vault TLS, OVN, Ceph, Octavia, and Magnum/CAPI tenant Kubernetes. Use this skill for ANY work touching OpenStack, Juju, MAAS, Magnum, CAPI, Ceph, OVN, Octavia, Keystone, Vault-for-OpenStack, tenant onboarding, or the repository this skill operates against - including writing or reviewing bash/python operational scripts, debugging failed deploys or cluster creates, runbook work, design-decision (D-NNN) discussion, and incident triage. Use it even for seemingly simple OpenStack questions: this deployment has strict operating discipline and known charm traps that make generic answers wrong." +--- + +# openstack-cloud-ops + +Operating skill for the Omega Cloud: a commercial, multi-tenant, tenant +self-administered OpenStack cloud, built as virtual rehearsals (VR0 single-DC +"testcloud"; VR1 two-DC + Office1 headend) for a future bare-metal +multi-datacenter deployment ("Roosevelt"). The CURRENT phase/stage lives in +repo `docs/CURRENT-STATE.md` ONLY (GA-R1) -- never trust a phase claim from +this file or memory. The governing design constraint is +MINIMIZE DELTA TO ROOSEVELT: the runbooks and scripts are primary deliverables +alongside the running cloud, so transferable answers beat quick fixes. + +## Step 0 - locate the source of truth + +This repository (GitBucket, git.baldurkeep.com) is authoritative for +everything: bundle, runbooks, scripts, design decisions, as-built values. This +skill is a discipline-and-routing layer OVER that repo, not a substitute for +it. Never hardcode the repo's own name/path in a command -- it has already been +renamed/forked once (see D-110) and will again; discover it fresh each session. + +1. Look for a local clone: a repo dir in the working tree, or a `$HOME` entry + containing `bundle.yaml` + `docs/design-decisions.md`. If found, confirm + identity with `git remote -v`, then `git log -1` to note HEAD and work from it. +2. No clone and you have shell + network: ask the operator for the exact clone + URL (org/group and repo name -- do not guess or reuse one from memory; it + may have moved, per this same repo's own history). The repo may be private; + if the clone fails, ask the operator to provide access or the relevant files. +3. No clone obtainable (e.g. chat without sandbox network): say so, ask the + operator to paste the relevant runbook/script, and proceed only on what is + actually in front of you. + +**Divergence rule:** if this skill and repo HEAD disagree, the repo wins - +but FLAG the divergence to the operator rather than silently following either. +The repo is a living draft; this skill's invariants (discipline, hardening) +change slowly, its facts (IPs, versions, phase status) go stale fast. + +**Session bookends (GA-R1/GA-R4).** OPEN: read `docs/CURRENT-STATE.md` in full +BEFORE any other doc (it is the only status authority), then `docs/session-ledger.md` ++ `bash scripts/ledger-scan.sh` to recover what is in flight and reconcile it against +repo ground truth (open decisions, open SEC count, next-free numbers). Re-do this +after any compaction. CLOSE: a bounded ledger summary (15-line hard cap, GA-R4 rule +1); the full session body goes to `docs/archive/` in the same close commit; commit +and push -- a session ending without its bookend is treated as NOT durable by the +next session. See `references/operating-discipline.md` (Session continuity). +## Step 1 - detect the environment + +- **Live shell to the jumphost / infra** (Claude Code on `vcloud` or + similar): you may RUN read-only audits directly. Every mutation remains + individually human-gated - present the command, state what it changes, wait + for approval. A live shell relaxes the transport, never the discipline. +- **Chat / no infra shell**: operate the gated copy-paste model - prepare + labeled blocks, the operator runs them and pastes output back. Never assume + a block ran or succeeded; wait for the pasted evidence. + +Read `references/operating-discipline.md` before doing either. + +## The three hard operating rules (non-negotiable) + +1. **Execute only the current runbook step, exactly as written.** No added + scope, no adjacent improvements, no live re-architecture mid-step. Findings + and improvement ideas are LOGGED (changelog / D-NNN proposal), never + executed live mid-step. +2. **Never use an inferred value.** No IP, ID, name, or scope goes into a + command unless it was measured this session or carried from confirmed + as-built. If a value would be inferred: stop and measure it. Never run a + destructive or session-altering command from memory without confirming it + is the minimal correct action for the current live state. +3. **Prefer dynamic lookups over hardcoded literals.** Discover VIPs, project + names, IDs, and version sets at runtime. Where a literal is unavoidable it + is tagged and centralized (`scripts/lib-net.sh`, `lib-hosts.sh`), keyed by + stable identity (CIDR, hostname) - never by drifting IDs. + +Corollary that governs everything: **verify before mutate**. A read-only audit +precedes every mutation; destructive and secret-handling steps are gated +individually, never batched. + +## Routing - where to go for what + +| Task | Read first | +|---|---| +| Any command block, script, or paste block you are about to write | `references/script-authoring.md` | +| Deploy / redeploy / teardown (VR0, single-DC testcloud) | repo `runbooks/README.md`, then the phase-NN runbook; conventions in `references/operating-discipline.md` | +| VR1 DC-DC buildout (two-DC + Office1 headend -- the current live mission) | repo `docs/CURRENT-STATE.md` FIRST (stage/gate STATUS lives there only), then `docs/dc-dc-deployment-workflow.md` (stage identity, links, gap register), then `runbooks/README.md`'s "VR1 DC-DC track" section for the `dc-dc-phase0..6` runbooks + `dc-dc-teardown-rollback.md`. SEPARATE track from VR0's phase-NN above -- do not conflate stage numbers between the two. | +| Something is broken (triage, incidents) | `references/troubleshooting.md`, then repo `runbooks/appendix-A-troubleshooting.md` | +| CAPI / Magnum / mgmt-VM recovery | repo `runbooks/ops-capi-recovery.md` | +| Deliver ANY repo change (script, runbook, doc) | run `bash scripts/repo-lint.sh` + the touched script's `tests//run-tests.sh` BEFORE handing it over | +| Pre-deploy gate (before add-model) | `bash scripts/preflight.sh` -- THE single entry; do not run the sub-gates piecemeal | +| Is the cloud actually healthy? (post-deploy, post-restart, pre-change baseline, incident) | `bash scripts/cloud-assert.sh` (add `--capture` at deploy completion for the committed BOM) | +| Full-cloud restart after outage/maintenance | repo `runbooks/ops-restart-procedure.md` | +| VR1 site unreachable after a vcloud reboot (`ssh voffice1` / `office1-*` time out) | `bash scripts/site-baseleg.sh check office1` -- `office1-local` is an ISOLATED libvirt net, so the host's base L3 leg (`10.10.0.10/24` + compose route `10.10.1.0/24 via 10.10.0.20`) is dropped every reboot; the `After=libvirtd` unit re-adds it (D-126 amendment / D-128). If MISS and the unit is not yet installed: `sudo bash scripts/site-baseleg.sh apply office1`. This is the vcloud path; the workstation reaches the same over the tailnet (D-107). | +| Starting any consequential live session | `bash scripts/run-logged.sh