#!/usr/bin/env python3
"""
provider-bundle-check.py -- focused, fail-closed QA for the Pattern A provider revert (D-060).
Asserts ONLY the post-revert (D-052/D-053 + Pattern A) provider invariants on a
Charmed-OpenStack bundle:
1. exactly 11 API charms bind public -> provider-public; none remain on provider-vip
2. every clustered VIP is EITHER a v4 triple -- provider-public(10.12.4/22)
admin(10.12.8/22) internal(10.12.12/22) -- OR a dual-family sextet appending
the three v6 legs (R2, RULED 2026-07-27: dual-stack for both DCs). All legs
share ONE host part, in 50-99 (D-020 amendment / R11, 2026-07-27), and the v6
host part MIRRORS the v4 octet textually (D-136 ruling, same date).
prefer-ipv6 and the v6 legs must land TOGETHER (see invariant 9).
3. ovn-chassis bridge-interface-mappings carries ALL FOUR chassis MACs, INCLUDING
openstack0's -- the Pattern A revert re-adds the openstack0 MAC that D-057 trimmed
for the now-dead provider-vip plane.
Structural invariants (absorbed from the retired scripts/review-bundle.py -- DOCFIX-070;
that linter's expectations were pre-D-052 and its NOT CLEAN verdict was pure noise):
4. every relation side names an EXISTING app and carries an explicit :endpoint
5. mysql-innodb-cluster deploys at num_units 3 (D-062 -- single-unit seed never bootstraps)
6. no VIP last-octet is shared between applications
7. keystone ships the D-051/D-064 policy IN the bundle (resources: policyd-override,
use-policyd-override true), and the committed zip content matches
policies/domain-manager-policy.yaml (DOCFIX-071 drift guard; content compare, not
byte compare, so zip mtimes cannot false-fail it)
8. R11 (D-020 AMENDMENT, 2026-07-27): a principal wired to an hacluster subordinate
MUST carry a vip. Without one it binds a UNIT address and pacemaker has nothing
to fail over to -- decorative HA. `cluster_count` is asserted NOWHERE in this
repo, so a 3->1 rewrite of every value produces a byte-identical PASS; this is
the ruled gate hardening that makes the shape checkable at all.
9. prefer-ipv6 and dual-family VIP arity are COUPLED. prefer-ipv6 makes HAProxy bind
:::port in ADDITION to *:port, so the two must travel together. Measured (L3-9):
the overlay merge order that keeps prefer-ipv6 while silently dropping the v6 VIP
legs is the one that exits 0 -- the dangerous order is the GREEN one.
It is the single deploy-gate: it REPLACED the retired scripts/d057-bundle-check.py (D-060)
and now also scripts/review-bundle.py (DOCFIX-070). FAIL -> exit 1. ASCII-only output.
(repo-lint: allow-stale-tokens -- guard checks name retired tokens by necessity)
"""
import sys, re, ipaddress, argparse
try:
import yaml
except ImportError:
sys.stderr.write("ERROR: PyYAML not installed (pip install pyyaml --break-system-packages)\n"); sys.exit(2)
# Per-DC VIP bands (D-101: only the network prefix moves per DC; the octet band is
# unchanged). --dc selects which set the VIP checks expect; default = the base
# bundle's dc0 values, so an un-parameterized run is byte-identical to before.
DC_BANDS = {
"vr1-dc0": ("10.12.4.0/22", "10.12.8.0/22", "10.12.12.0/22"),
"vr1-dc1": ("10.12.64.0/22", "10.12.68.0/22", "10.12.72.0/22"),
}
# D-020 AMENDMENT / R11 (RULED 2026-07-27): the band widens 50-60 -> 50-99 to admit
# vault .61 and designate .62. EXPECT_PUBLIC_VIP deliberately STAYS 11 -- measured,
# NEITHER vault nor designate carries a `public` binding, so they do not join that
# count and bumping both constants together breaks the gate.
OCTET_LO, OCTET_HI = 50, 99
EXPECT_PUBLIC_VIP = 11
# v6 VIP bands are NOT hardcoded: D-136 option (D) (ADOPTED 2026-07-27) makes the
# NetBox apex the source for v6 prefixes. Same record and same (role, kind) keying
# as scripts/dc-plane-ipam.sh:102-131 and netbox/dc-plane-apex-import.py.
APEX_ENV = "APEX_RECORD"
APEX_GLOB = "netbox/draft/vr1-office1-current-*.json"
EXPECT_CHASSIS_MACS = {
"52:54:00:3d:fd:54", # openstack0 -- re-added by the Pattern A revert (D-057 had trimmed it)
"52:54:00:9d:63:77", # openstack1
"52:54:00:89:7f:ce", # openstack2
"52:54:00:99:fc:c2", # openstack3
}
MAC_RE = re.compile(r"[0-9a-f]{2}(?::[0-9a-f]{2}){5}")
ROLES = ("control", "compute", "storage")
def _deep_merge(a, b):
if isinstance(a, dict) and isinstance(b, dict):
out = dict(a)
for k, v in b.items():
out[k] = _deep_merge(a[k], v) if k in a else v
return out
return b
def merge_overlay(base, over):
"""Juju-like overlay merge: applications/machines deep-merge key-by-key (a None
overlay value deletes that key), relations append, other top-level keys override.
NOTE: this MIRRORS juju's documented merge so the checker validates the effective
deploy input; the machines-block merge in particular is CONFIRMED at
`juju deploy --dry-run` before deploy (no offline juju merger exists)."""
base, over = base or {}, over or {}
out = dict(base)
for section in ("applications", "machines"):
b = dict(base.get(section) or {})
for k, v in (over.get(section) or {}).items():
if v is None:
b.pop(k, None)
elif isinstance(b.get(k), dict) and isinstance(v, dict):
b[k] = _deep_merge(b[k], v)
else:
b[k] = v
if b:
out[section] = b
rels = list(base.get("relations") or []) + list(over.get("relations") or [])
if rels:
out["relations"] = rels
for k, v in over.items():
if k not in ("applications", "machines", "relations"):
out[k] = v
return out
def _apex_v6_bands(dc):
"""The three v6 VIP-leg /64s for `dc`, read from the NetBox apex record.
Returns (bands, None) on success or (None, reason) -- a reason is a REFUSAL
(exit 2), never a pass: "could not look" is never "nothing there".
The PROVIDER leg lives in the DEDICATED GUA VIP /64 (the apex marks it with
"VIP" in the description), NOT the provider node /64; admin and internal use
their own plane /64s. The record's shape differs between the live API and the
repo dumps (live returns nested scope.slug/scope.name, dumps flatten to
scope_site + a bare role string), so both are handled -- writing against one
shape and matching zero objects on the other is a measured trap.
"""
import glob, json, os
rec = os.environ.get(APEX_ENV)
if not rec:
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
cand = sorted(glob.glob(os.path.join(root, APEX_GLOB)))
rec = cand[-1] if cand else None
if not rec or not os.path.isfile(rec):
return None, "no readable apex record (looked for %s; set %s)" % (APEX_GLOB, APEX_ENV)
try:
doc = json.load(open(rec, encoding="utf-8"))
except Exception as e:
return None, "apex record %s unreadable: %s" % (rec, e)
found = {}
for p in (doc.get("ipam/prefixes") or []):
pre = str(p.get("prefix", ""))
if ":" not in pre or not pre.endswith("/64"):
continue # /60 and /48 are parent blocks, not planes
sc = p.get("scope") or {}
if (sc.get("slug") or sc.get("name") or p.get("scope_site") or "") != dc:
continue
role = p.get("role") or ""
role = role.get("slug") if isinstance(role, dict) else role
kind = "vip" if "VIP" in str(p.get("description") or "") else "plane"
found[(role, kind)] = pre
want = (("provider", ("provider-public", "vip")),
("admin", ("metal-admin", "plane")),
("internal", ("metal-internal", "plane")))
bands, missing = {}, []
for leg, key in want:
if key in found:
bands[leg] = ipaddress.ip_network(found[key])
else:
missing.append("%s/%s" % key)
if missing:
return None, ("apex record %s carries no v6 prefix for %s in %s -- refusing to "
"guess a VIP band" % (rec, ", ".join(missing), dc))
return bands, None
def _host_part(addr):
"""The trailing label of an address: the v4 last octet, or the final v6 group.
The 2026-07-27 ruling mirrors the v4 octet DIGITS into the v6 host part
(keystone .50 -> ...::50), so the two are compared as TEXT and never
numerically -- `printf '%x' 50` is 32, which is exactly the plausible-looking
wrong band that ruling exists to avoid.
"""
ip = ipaddress.ip_address(addr)
s = str(ip)
return s.rsplit(".", 1)[-1] if ip.version == 4 else s.rsplit(":", 1)[-1]
def _role_of(spec):
"""Extract control/compute/storage from a machine spec's `constraints: tags=...`."""
m = re.search(r"tags=(\S+)", str((spec or {}).get("constraints", "")))
tags = set(m.group(1).split(",")) if m else set()
for r in ROLES:
if r in tags:
return r
return None
def main():
ap = argparse.ArgumentParser(description="Pattern A / D-052-D-053 + placement bundle invariants")
ap.add_argument("path", nargs="?", default="bundle.yaml", help="base bundle (default bundle.yaml)")
ap.add_argument("--overlay", action="append", default=[], metavar="FILE",
help="overlay bundle(s), applied in order (juju-like merge) to validate the effective deploy input")
ap.add_argument("--dc", choices=sorted(DC_BANDS), default="vr1-dc0",
help="DC whose VIP bands to expect (default vr1-dc0 = the base bundle's own values)")
args = ap.parse_args()
path = args.path
try:
doc = yaml.safe_load(open(path, encoding="utf-8"))
except Exception as e:
sys.stderr.write("ERROR: cannot parse %s: %s\n" % (path, e)); return 2
for ov in args.overlay:
try:
doc = merge_overlay(doc, yaml.safe_load(open(ov, encoding="utf-8")))
except Exception as e:
sys.stderr.write("ERROR: cannot merge overlay %s: %s\n" % (ov, e)); return 2
PROVIDER, ADMIN, INTERNAL = (ipaddress.ip_network(x) for x in DC_BANDS[args.dc])
apps = (doc or {}).get("applications", {}) or {}
machines = (doc or {}).get("machines", {}) or {}
roles = {str(mid): _role_of(spec) for mid, spec in machines.items()}
role_sep = any(roles.values()) # role-separated (VR1) vs hyperconverged base (VR0)
fails, oks = [], []
def pub(s): return ((s or {}).get("bindings", {}) or {}).get("public")
on_vip = sorted(n for n, s in apps.items() if pub(s) == "provider-vip")
on_public = sorted(n for n, s in apps.items() if pub(s) == "provider-public")
if on_vip:
fails.append("public still on provider-vip (must be reverted to provider-public): %s" % ", ".join(on_vip))
if len(on_public) != EXPECT_PUBLIC_VIP:
fails.append("public->provider-public count=%d (expect %d): %s" % (len(on_public), EXPECT_PUBLIC_VIP, ", ".join(on_public)))
else:
oks.append("%d charms bind public->provider-public; none on provider-vip" % len(on_public))
# -- 2 + 9. VIP legs, arity, family coupling, band, and octet uniqueness.
# A vip is a v4 TRIPLE (3) or, under R2's ruled dual-stack, a dual-family
# SEXTET (6) appending the v6 provider/admin/internal legs.
v6_bands, v6_refusal = None, None
vip_ok = vip_dual = 0
octet_owner = {}
for n in sorted(apps):
opts = ((apps[n] or {}).get("options") or {})
vip = opts.get("vip")
if not vip:
continue
prefer6 = bool(opts.get("prefer-ipv6"))
parts = str(vip).split()
if len(parts) not in (3, 6):
fails.append("%s vip is neither a v4 triple (3) nor a dual-family sextet (6) "
"-- got %d address(es): %r" % (n, len(parts), vip)); continue
dual = (len(parts) == 6)
# prefer-ipv6 binds :::port in ADDITION to *:port, so it and the v6 legs must
# land together. Measured (L3-9): the overlay merge order that keeps
# prefer-ipv6 while dropping the v6 legs is the one that exits 0 -- pacemaker
# would then have no v6 VIP to manage behind a socket that is already bound.
if prefer6 != dual:
fails.append("%s prefer-ipv6=%s but its vip carries %d address(es) -- "
"prefer-ipv6 and the v6 VIP legs must land TOGETHER"
% (n, str(prefer6).lower(), len(parts))); continue
if dual and v6_bands is None and v6_refusal is None:
v6_bands, v6_refusal = _apex_v6_bands(args.dc)
if dual and v6_bands is None:
sys.stderr.write("ERROR: cannot evaluate %s's dual-family vip: %s\n" % (n, v6_refusal))
return 2
legs = [("provider", parts[0], PROVIDER),
("admin", parts[1], ADMIN),
("internal", parts[2], INTERNAL)]
if dual:
legs += [("provider v6", parts[3], v6_bands["provider"]),
("admin v6", parts[4], v6_bands["admin"]),
("internal v6", parts[5], v6_bands["internal"])]
bad = False
for lbl, addr, net in legs:
try:
ip = ipaddress.ip_address(addr)
except ValueError as e:
fails.append("%s bad vip ip: %s" % (n, e)); bad = True; break
if ip.version != net.version:
fails.append("%s %s leg %s is IPv%d, expected IPv%d"
% (n, lbl, addr, ip.version, net.version)); bad = True; break
if ip not in net:
fails.append("%s %s leg %s not in %s" % (n, lbl, addr, net)); bad = True; break
if bad:
continue
hosts = {_host_part(p) for p in parts}
if len(hosts) != 1:
fails.append("%s vip host parts differ across legs -- the v6 host part must "
"MIRROR the v4 octet (2026-07-27 ruling): %r" % (n, vip)); continue
h = hosts.pop()
if not h.isdigit():
fails.append("%s vip host part %r is not the decimal v4-octet mirror" % (n, h)); continue
o = int(h)
if not (OCTET_LO <= o <= OCTET_HI):
fails.append("%s vip octet .%d outside %d-%d" % (n, o, OCTET_LO, OCTET_HI)); continue
if h in octet_owner:
fails.append("VIP last octet .%s shared by %s and %s" % (h, octet_owner[h], n)); continue
octet_owner[h] = n
vip_ok += 1
vip_dual += 1 if dual else 0
if vip_ok:
oks.append("%d clustered VIP(s) are provider/admin/internal, octet %d-%d "
"(%d dual-family)" % (vip_ok, OCTET_LO, OCTET_HI, vip_dual))
for n, s in apps.items():
if (s or {}).get("charm") != "ovn-chassis":
continue
bim = str(((s or {}).get("options", {}) or {}).get("bridge-interface-mappings", ""))
if not bim:
continue
macs = set(MAC_RE.findall(bim.lower()))
if role_sep:
# VR1 role-separated: the VR0 openstack0-3 Pattern-A set (D-057/D-060) does not
# apply -- dc chassis are the compute-node provider MACs. Verify present + well-formed.
if macs:
oks.append("%s bridge-interface-mappings: %d well-formed MAC(s) (role-sep; VR0 set N/A)" % (n, len(macs)))
else:
fails.append("%s bridge-interface-mappings has no valid MAC" % n)
else:
missing = EXPECT_CHASSIS_MACS - macs
if missing:
fails.append("%s missing chassis MAC(s): %s" % (n, ", ".join(sorted(missing))))
else:
oks.append("%s bridge-interface-mappings: all 4 chassis MACs present (incl openstack0)" % n)
# -- 4. relations: existing apps, explicit endpoints (the magnum-shared-db class) --
rels = (doc or {}).get("relations", []) or []
rel_bad = 0
for r in rels:
if not isinstance(r, list) or len(r) != 2:
fails.append("relation not a 2-list: %r" % (r,)); rel_bad += 1; continue
for side in r:
side = str(side)
if ":" not in side:
fails.append("relation side lacks explicit :endpoint: %r" % side); rel_bad += 1; continue
if side.split(":")[0] not in apps:
fails.append("relation references unknown app: %r" % side); rel_bad += 1
if rels and not rel_bad:
oks.append("%d relations well-formed (explicit endpoints, all apps exist)" % len(rels))
# -- 5. D-062: mysql-innodb-cluster at target count 3 --
mi = apps.get("mysql-innodb-cluster") or {}
if mi.get("num_units") != 3:
fails.append("mysql-innodb-cluster num_units=%r (D-062 requires 3: single-unit seed never bootstraps)" % mi.get("num_units"))
else:
oks.append("mysql-innodb-cluster num_units=3 (D-062)")
# -- 6. VIP octet uniqueness: folded into the family-aware loop above, which
# compares the HOST PART (v4 octet or final v6 group) rather than
# rsplit(".")-ing every token -- on a v6 literal that returned the whole
# address, so every v6 leg was trivially "unique" and unchecked.
# -- 8. R11 (D-020 AMENDMENT, 2026-07-27): an hacluster principal with no VIP.
# Decorative HA -- the principal binds a UNIT address and pacemaker has
# nothing to fail over to. Nothing in scripts/ or tests/ asserts
# cluster_count, so this shape passed every gate until now.
ha_principals = set()
for r in rels:
if not isinstance(r, list) or len(r) != 2:
continue
sides = [str(x) for x in r]
if not all(x.endswith(":ha") for x in sides):
continue
for i, side in enumerate(sides):
app, peer = side.split(":")[0], sides[1 - i].split(":")[0]
if app in apps and (apps.get(peer) or {}).get("charm") == "hacluster":
ha_principals.add(app)
no_vip = sorted(a for a in ha_principals
if not ((apps.get(a) or {}).get("options") or {}).get("vip"))
if no_vip:
fails.append("hacluster relation but no vip: %s -- clustered principal binds a unit "
"address with nothing to fail over to (R11 / D-020 amendment)"
% ", ".join(no_vip))
elif ha_principals:
oks.append("%d hacluster principal(s) all carry a VIP (R11)" % len(ha_principals))
# -- 7. DOCFIX-071: keystone policy ships in-bundle, zip content matches source --
import os, zipfile
ks = apps.get("keystone") or {}
res = ((ks.get("resources") or {}).get("policyd-override"))
upo = ((ks.get("options") or {}).get("use-policyd-override"))
if not upo:
fails.append("keystone use-policyd-override is not true (D-051)")
if not res:
fails.append("keystone has no resources: policyd-override (DOCFIX-071: policy unreachable on redeploy)")
else:
base = os.path.dirname(os.path.abspath(path))
zp = os.path.normpath(os.path.join(base, str(res)))
src = os.path.join(base, "policies", "domain-manager-policy.yaml")
if not os.path.isfile(zp):
fails.append("policyd-override zip missing at %s" % zp)
elif not os.path.isfile(src):
fails.append("policy source missing at %s" % src)
else:
try:
with zipfile.ZipFile(zp) as z:
inzip = z.read("domain-manager-policy.yaml")
ondisk = open(src, "rb").read()
if inzip != ondisk:
fails.append("policyd-override zip content DIFFERS from policies/domain-manager-policy.yaml (rebuild + recommit the zip)")
else:
oks.append("keystone policyd-override wired in-bundle; zip content matches source (DOCFIX-071)")
except KeyError:
fails.append("zip lacks top-level domain-manager-policy.yaml (keystone reads the top-level name)")
except Exception as e:
fails.append("cannot read policyd zip: %s" % e)
# -- 8. Placement / anti-affinity (role separation). Activates ONLY when the
# machines block carries control/compute/storage role tags; a base or
# un-rendered bundle has none, so the checks self-skip (base stays green).
# Run on the MERGED bundle (base + overlays) to see the scaled apps' real
# num_units/to:. Catches the decorative-HA paths juju itself won't error on.
if role_sep:
pf0 = len(fails)
n_ctl = sum(1 for r in roles.values() if r == "control")
n_cmp = sum(1 for r in roles.values() if r == "compute")
n_stg = sum(1 for r in roles.values() if r == "storage")
for n, s in apps.items():
s = s or {}
tos = [str(t) for t in (s.get("to") or [])]
if not tos:
continue
ids = [t.split(":")[-1] for t in tos]
for t, mid in zip(tos, ids): # a. no dangling placement
if mid not in roles:
fails.append("%s to: references undefined machine %r" % (n, t))
want = ("storage" if n == "ceph-osd" else "compute" if n == "nova-compute"
else "control" if any(t.startswith("lxd:") for t in tos) else None)
if want: # b. role placement
off = [t for t, mid in zip(tos, ids) if roles.get(mid) != want]
if off:
fails.append("%s placed off-role (want %s node): %s" % (n, want, ", ".join(off)))
nu = s.get("num_units") # d. anti-affinity
if isinstance(nu, int) and nu >= 2 and len(set(ids)) < nu:
fails.append("%s num_units=%d but to: has %d distinct machine(s) -- anti-affinity defeat: %s"
% (n, nu, len(set(ids)), ids))
for n, want, lbl in (("ceph-osd", n_stg, "storage"), ("nova-compute", n_cmp, "compute")):
nu = (apps.get(n) or {}).get("num_units") # c. bare-metal role counts
if isinstance(nu, int) and nu != want:
fails.append("%s num_units=%d != %d %s node(s)" % (n, nu, want, lbl))
if len(fails) == pf0:
oks.append("placement: role-separated (%d control/%d compute/%d storage); "
"anti-affinity + role placement + counts OK" % (n_ctl, n_cmp, n_stg))
else:
oks.append("placement: no role tags in machines block (base/un-rendered bundle) -- checks skipped")
for o in oks: print(" [ok] %s" % o)
for f in fails: print(" [FAIL] %s" % f)
print("\n%s: Pattern A / D-052-D-053 bundle invariants (%s)" % ("PASS" if not fails else "FAIL", path))
return 1 if fails else 0
if __name__ == "__main__":
sys.exit(main())
