Newer
Older
openstack-caracal-dc-dc / tests / provider-bundle-check / run-tests.sh
#!/usr/bin/env bash
# tests/provider-bundle-check/run-tests.sh
#
# Offline regression harness for scripts/provider-bundle-check.py. No live infra;
# fixtures are generated from the repo bundle by targeted mutation, so every check
# is proven to FAIL when its invariant is violated (bash -n / "it parsed" proves
# nothing -- behavior is the contract). Mutates NOTHING outside $TMP.
# Exit: 0 all pass | 1 any case failed.  ASCII + LF.
set -uo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO="$(cd "$HERE/../.." && pwd)"
CHK="$REPO/scripts/provider-bundle-check.py"
TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT
PASS=0; FAIL=0

# fixture base: the real dc0 DEPLOY INPUT + the policies dir DOCFIX-071 resolves.
#
# `pristine.yaml` is bundle.yaml MERGED WITH overlays/vr1-dc0-vips.yaml -- since
# ruling 3 (2026-07-25) the base is VIP-free and the merged pair is what actually
# deploys. `good.yaml` is that plus the ONE thing R11 (D-020 amendment, RULED
# 2026-07-27) ruled and that is not built yet: designate's VIP at the ruled octet
# .62. Without it the new hacluster-principal check fails the real input, and every
# rc=0 case below would assert "fails for the reason we already know" instead of its
# own invariant. T16 keeps the real tree honest by asserting the merged input DOES
# trip it; T31 asserts the bare base is no longer a deploy input at all.
mkdir -p "$TMP/policies"
cp "$REPO/policies/domain-manager-policy.yaml" "$REPO/policies/overrides.zip" "$TMP/policies/"
# RULING 3 (2026-07-25) made bundle.yaml VIP-FREE: every VIP arrives via a per-DC
# overlay, dc0 INCLUDED. So the dc0 DEPLOY INPUT -- and therefore the fixture base
# -- is base + overlays/vr1-dc0-vips.yaml. The bare base is no longer a deploy
# input at all, and T31 asserts exactly that rather than leaving it untested.
python3 - "$REPO/bundle.yaml" "$REPO/overlays/vr1-dc0-vips.yaml" \
         "$TMP/pristine.yaml" "$TMP/good.yaml" "$TMP/v4only.yaml" <<'PY'
import sys, yaml, copy
base = yaml.safe_load(open(sys.argv[1]))
ovl  = yaml.safe_load(open(sys.argv[2]))
for n, s in (ovl.get("applications") or {}).items():
    base["applications"].setdefault(n, {}).setdefault("options", {}).update(
        (s or {}).get("options") or {})
yaml.safe_dump(base, open(sys.argv[3], "w"))
# Since R11 landed (2026-07-29) the overlay supplies designate AND vault, so the
# deploy input needs no injection any more -- good == pristine. The case that used
# to assert the .62 gap is RE-POINTED below, not deleted.
yaml.safe_dump(base, open(sys.argv[4], "w"))
# A V4-ONLY twin: every vip truncated to its three v4 legs and prefer-ipv6 dropped,
# TOGETHER (dropping one alone IS the L3-9 defect). The v4-shaped cases mutate this
# instead of good.yaml, so they still exercise the band / octet-uniqueness /
# apex-independence invariants they were written for, rather than tripping the
# prefer-ipv6 coupling check first and never reaching them.
v4 = copy.deepcopy(base)
for n, spec in v4["applications"].items():
    o = (spec or {}).get("options") or {}
    if o.get("vip"):
        o["vip"] = " ".join(str(o["vip"]).split()[:3])
        o.pop("prefer-ipv6", None)
yaml.safe_dump(v4, open(sys.argv[5], "w"))
PY
# A valid dual-family (R2 dual-stack) VIP for keystone: the three v4 legs plus the
# three v6 legs, host part mirroring the v4 octet per the 2026-07-27 ruling. The
# provider v6 leg is the DEDICATED GUA VIP /64 (f02:11::), not the node /64.
DUAL6="10.12.4.50 10.12.8.50 10.12.12.50 2602:f3e2:f02:11::50 fd50:840e:74e2:220::50 fd50:840e:74e2:221::50"

mutate4() { # mutate the V4-ONLY twin -- for cases whose invariant predates dual-stack
  python3 - "$TMP/v4only.yaml" "$TMP/$1" "$2" <<'PY'
import sys, yaml
src, dst, expr = sys.argv[1], sys.argv[2], sys.argv[3]
b = yaml.safe_load(open(src))
exec(expr)
yaml.safe_dump(b, open(dst, "w"))
PY
}

mutate() { # mutate <out> <python-expr operating on dict b>
  python3 - "$TMP/good.yaml" "$TMP/$1" "$2" <<'PY'
import sys, yaml
src, dst, expr = sys.argv[1], sys.argv[2], sys.argv[3]
b = yaml.safe_load(open(src))
exec(expr)
yaml.safe_dump(b, open(dst, "w"))
PY
}

run() { # run <want_rc> <regex> <label> <bundle>
  local want="$1" rx="$2" label="$3" f="$4"
  local out rc
  out="$(python3 "$CHK" "$f" 2>&1)"; rc=$?
  if [[ "$rc" == "$want" ]] && grep -qE "$rx" <<<"$out"; then
    echo "  PASS  $label"; PASS=$((PASS+1))
  else
    echo "  FAIL  $label (rc=$rc want=$want)"; echo "$out" | sed 's/^/        /' | head -6; FAIL=$((FAIL+1))
  fi
}

# T1 happy path: the repo bundle itself must PASS
run 0 'PASS: Pattern A' "T1 repo bundle passes clean" "$TMP/good.yaml"

# T2 relation referencing a ghost app
mutate t2.yaml 'b["relations"].append(["ghost-app:amqp","rabbitmq-server:amqp"])'
run 1 'unknown app' "T2 ghost relation app FAILS" "$TMP/t2.yaml"

# T3 endpoint-less relation side (the missing-colon class)
mutate t3.yaml 'b["relations"].append(["keystone","rabbitmq-server:amqp"])'
run 1 'lacks explicit :endpoint' "T3 endpoint-less relation FAILS" "$TMP/t3.yaml"

# T4 D-062 regression: mysql at 1 unit
mutate t4.yaml 'b["applications"]["mysql-innodb-cluster"]["num_units"]=1'
run 1 'D-062 requires 3' "T4 mysql num_units=1 FAILS" "$TMP/t4.yaml"

# T5 shared VIP octet between two apps
mutate4 t5.yaml 'b["applications"]["glance"]["options"]["vip"]="10.12.4.50 10.12.8.50 10.12.12.50"'
run 1 'shared by' "T5 duplicate VIP octet FAILS" "$TMP/t5.yaml"

# T6 DOCFIX-071 regression: keystone resource stanza removed
mutate t6.yaml 'b["applications"]["keystone"].pop("resources",None)'
run 1 'DOCFIX-071' "T6 missing policyd resource FAILS" "$TMP/t6.yaml"

# T7 DOCFIX-071 drift: zip content differs from the yaml source
mutate t7.yaml 'pass'
printf '\n# drift\n' >> "$TMP/policies/domain-manager-policy.yaml"
run 1 'DIFFERS from policies' "T7 zip/source drift FAILS" "$TMP/t7.yaml"
cp "$REPO/policies/domain-manager-policy.yaml" "$TMP/policies/"   # restore

# T8 original invariant still guarded: octavia VIP off-band (band widened to 50-99 by R11)
mutate4 t8.yaml 'b["applications"]["octavia"]["options"]["vip"]="10.12.4.233 10.12.8.233 10.12.12.233"'
run 1 'outside 50-99' "T8 off-band VIP octet FAILS" "$TMP/t8.yaml"

# ---- Placement / anti-affinity checks (only fire on a role-separated bundle) ----
# Build a valid role-separated fixture from the base: 3 control / 2 compute / 4 storage,
# quorum trio + control containers on control, ceph-osd on storage, nova-compute on compute.
python3 - "$TMP/good.yaml" "$TMP/rolesep.yaml" <<'PY'
import sys, yaml
b = yaml.safe_load(open(sys.argv[1]))
roles = ["control","control","control","compute","compute","storage","storage","storage","storage"]
b["machines"] = {str(i): {"constraints": "arch=amd64 tags=openstack-vr1-dc0,%s" % r}
                 for i, r in enumerate(roles)}
for n, s in b["applications"].items():
    if not isinstance(s, dict) or "to" not in s:
        continue
    if   n == "ceph-osd":     s["num_units"] = 4; s["to"] = ["5","6","7","8"]
    elif n == "nova-compute": s["num_units"] = 2; s["to"] = ["3","4"]
    elif n in ("mysql-innodb-cluster","ovn-central","ceph-mon"): s["to"] = ["lxd:0","lxd:1","lxd:2"]
    else:                     s["to"] = ["lxd:0"]
yaml.safe_dump(b, open(sys.argv[2], "w"))
PY

mutate_rs() { python3 - "$TMP/rolesep.yaml" "$TMP/$1" "$2" <<'PY'
import sys, yaml
b = yaml.safe_load(open(sys.argv[1])); exec(sys.argv[3]); yaml.safe_dump(b, open(sys.argv[2], "w"))
PY
}
runargs() { # runargs <want_rc> <regex> <label> -- <checker args...>
  local want="$1" rx="$2" label="$3"; shift 3
  local out rc; out="$(python3 "$CHK" "$@" 2>&1)"; rc=$?
  if [[ "$rc" == "$want" ]] && grep -qE "$rx" <<<"$out"; then
    echo "  PASS  $label"; PASS=$((PASS+1))
  else
    echo "  FAIL  $label (rc=$rc want=$want)"; echo "$out" | sed 's/^/        /' | head -6; FAIL=$((FAIL+1))
  fi
}

# T9 valid role-separated bundle PASSES (placement active, all invariants hold)
run 0 'counts OK' "T9 valid role-separated bundle passes" "$TMP/rolesep.yaml"

# T10 dangling to: (undefined machine)
mutate_rs t10.yaml 'b["applications"]["glance"]["to"]=["lxd:99"]'
run 1 'undefined machine' "T10 dangling to: FAILS" "$TMP/t10.yaml"

# T11 off-role: ceph-osd on control/compute instead of storage
mutate_rs t11.yaml 'b["applications"]["ceph-osd"]["to"]=["0","1","2","3"]'
run 1 'off-role .want storage' "T11 ceph-osd off-role FAILS" "$TMP/t11.yaml"

# T12 anti-affinity defeat: a 3-unit app stacked on 2 machines
mutate_rs t12.yaml 'b["applications"]["mysql-innodb-cluster"]["to"]=["lxd:0","lxd:0","lxd:1"]'
run 1 'anti-affinity defeat' "T12 stacked control units FAIL" "$TMP/t12.yaml"

# T13 wrong bare-metal count: ceph-osd != number of storage nodes
mutate_rs t13.yaml 'b["applications"]["ceph-osd"]["num_units"]=3'
run 1 '!= 4 storage node' "T13 ceph-osd count mismatch FAILS" "$TMP/t13.yaml"

# T14 overlay merge + DC bands: the REAL dc1 deploy input, --dc vr1-dc1 PASSES.
# RE-POINTED 2026-07-29: this used to apply the dc1 VIP overlay ALONE, which pinned
# "dc1 VIPs over dc0-tagged machines is fine". Invariant 11 rules that shape out --
# it is the 4.3 defect (a dc1 model allocating dc0 hardware) -- so the case is moved
# to the input that actually deploys (machines overlay INCLUDED) and the old shape
# becomes T32's RED fixture rather than being deleted.
runargs 0 'PASS: Pattern A' "T14 real dc1 deploy input (machines+vips) + --dc vr1-dc1 passes" \
  "$TMP/good.yaml" --overlay "$REPO/overlays/vr1-dc1-machines.yaml" \
  --overlay "$REPO/overlays/vr1-dc1-vips.yaml" --dc vr1-dc1

# T15 same overlay but default (dc0) bands -> dc1 VIPs off-band FAIL
runargs 1 'not in 10.12.4' "T15 dc1 VIPs vs dc0 bands FAILS" \
  "$TMP/good.yaml" --overlay "$REPO/overlays/vr1-dc1-vips.yaml"

# ---- R11 (D-020 amendment): an hacluster principal with no VIP is decorative HA ----
# T16 the REAL repo bundle trips it. This is the ruled-but-not-built designate VIP;
#     when .62 lands in bundle.yaml this case must be re-pointed, NOT deleted.
# RE-POINTED 2026-07-29: this asserted designate's VIP was MISSING. R11 built it, so
# the assertion is inverted to the surviving invariant -- the real deploy input is now
# CLEAN -- and T16b keeps the check's teeth by removing a VIP and demanding it fire.
run 0 '12 hacluster principal\(s\) all carry a VIP' \
  "T16 the real dc0 deploy input is CLEAN (R11 built)" "$TMP/pristine.yaml"

mutate t16b.yaml 'b["applications"]["designate"]["options"].pop("vip"); b["applications"]["designate"]["options"].pop("prefer-ipv6",None)'
run 1 'hacluster relation but no vip: designate' \
  "T16b removing designate's VIP still FAILS" "$TMP/t16b.yaml"

# T31 ruling 3's new invariant: the BARE base is not a deploy input. Validating it
#     alone must name every clustered principal, not silently pass -- a VIP-free
#     bundle deployed without its overlay is 11 charms of decorative HA.
run 1 'hacluster relation but no vip: barbican' \
  "T31 the VIP-free base ALONE fails for all clustered principals" "$REPO/bundle.yaml"

# T17 vault is the second R11 gap and it only appears under the HA overlay -- R6 ruled
#     the VIPs land BEFORE that overlay, so this is the ordering made executable.
# RE-POINTED 2026-07-29: R6 ruled the VIPs land BEFORE the HA overlay. They have, so
# the ordering is now SATISFIED and this asserts that -- 13 principals once the overlay
# adds vault-hacluster. T17b proves the check still fires if vault's VIP goes missing.
runargs 0 '13 hacluster principal\(s\) all carry a VIP' \
  "T17 R6 ordering satisfied: VIPs precede dc-ha-scaleup" \
  "$TMP/good.yaml" --overlay "$REPO/overlays/dc-ha-scaleup.yaml"

mutate t17b.yaml 'b["applications"]["vault"]["options"].pop("vip"); b["applications"]["vault"]["options"].pop("prefer-ipv6",None)'
runargs 1 'hacluster relation but no vip: .*vault' \
  "T17b removing vault's VIP under the HA overlay still FAILS" \
  "$TMP/t17b.yaml" --overlay "$REPO/overlays/dc-ha-scaleup.yaml"

# T18 the check can also PASS, not only fail: give designate its ruled VIP -> green.
run 0 'hacluster principal\(s\) all carry a VIP' \
  "T18 hacluster check goes GREEN once the ruled VIP lands" "$TMP/good.yaml"

# ---- R2 dual-stack: VIP arity, family coupling, and the apex-derived v6 bands ----
# T19 a valid dual-family sextet PASSES and is counted as dual
run 0 '13 clustered VIP\(s\).*13 dual-family' \
  "T19 every app is dual-family in the real deploy input" "$TMP/good.yaml"

# T20 prefer-ipv6 kept while the v6 legs are dropped -- the L3-9 merge order that
#     exits 0 today. This is the whole reason the coupling exists.
mutate t20.yaml 'o=b["applications"]["keystone"]["options"]; o["vip"]=" ".join(o["vip"].split()[:3])'
run 1 'prefer-ipv6 and the v6 VIP legs must land TOGETHER' \
  "T20 prefer-ipv6 without v6 legs FAILS" "$TMP/t20.yaml"

# T21 the inverse: v6 legs present but nothing binds v6
mutate t21.yaml 'b["applications"]["keystone"]["options"].pop("prefer-ipv6")'
run 1 'prefer-ipv6 and the v6 VIP legs must land TOGETHER' \
  "T21 v6 legs without prefer-ipv6 FAILS" "$TMP/t21.yaml"

# T22 v6 provider leg in the NODE /64 instead of the dedicated GUA VIP /64
mutate t22.yaml "b['applications']['keystone']['options'].update({'vip':'${DUAL6/f02:11::50/f02:10::50}','prefer-ipv6':True})"
run 1 'provider v6 leg .* not in 2602:f3e2:f02:11::/64' \
  "T22 v6 leg in the wrong /64 FAILS" "$TMP/t22.yaml"

# T23 v6 host part not mirroring the v4 octet -- `printf '%x' 50` = 32 is the exact
#     plausible-looking wrong value the 2026-07-27 ruling exists to prevent.
mutate t23.yaml "b['applications']['keystone']['options'].update({'vip':'${DUAL6/f02:11::50/f02:11::32}','prefer-ipv6':True})"
run 1 'must MIRROR the v4 octet' "T23 v6 host part not mirroring the v4 octet FAILS" "$TMP/t23.yaml"

# T24 arity that is neither a triple nor a sextet
mutate t24.yaml 'b["applications"]["keystone"]["options"]["vip"]="10.12.4.50 10.12.8.50"'
run 1 'neither a v4 triple .3. nor a dual-family sextet' "T24 2-address vip FAILS" "$TMP/t24.yaml"

# T25 an unreadable apex REFUSES (rc=2) rather than passing -- "could not look" is
#     never "nothing there". Only fires when a dual-family vip is actually present.
APEX_RECORD=/nonexistent/apex.json runargs 2 'cannot evaluate .* no readable apex record' \
  "T25 dual-family + unreadable apex REFUSES (rc=2)" "$TMP/good.yaml"

# T26 ...and a v4-only bundle does NOT need the apex at all (no spurious refusal)
APEX_RECORD=/nonexistent/apex.json runargs 0 'PASS: Pattern A' \
  "T26 v4-only bundle needs no apex record" "$TMP/v4only.yaml"

# ---- R11 band widen 50-60 -> 50-99 ----
# T27 octet .62 (designate's ruled octet) is now IN band -- proven by the band message
run 0 'octet 50-99' "T27 R11 band widened to 50-99" "$TMP/good.yaml"

# T28 ...but the band is still bounded: .100 is still rejected
mutate4 t28.yaml 'b["applications"]["glance"]["options"]["vip"]="10.12.4.100 10.12.8.100 10.12.12.100"'
run 1 'outside 50-99' "T28 band still bounded (.100 rejected)" "$TMP/t28.yaml"

# ---- the dc1 arm of the apex lookup, exercised for real ----
# T19-T23 all run at the default --dc vr1-dc0, so the dc1 v6 bands (f03:11::/64,
# :320::/64, :321::/64) were resolved by NO case. The renderer's first real output
# is per-DC overlays; a gate observed in only one DC mode is exactly the
# only-ever-seen-one-way untrustworthiness T16/T18 exist to rule out.
cat > "$TMP/dc1-dual.yaml" <<'YAML'
applications:
  keystone:
    options:
      prefer-ipv6: true
      vip: "10.12.64.50 10.12.68.50 10.12.72.50 2602:f3e2:f03:11::50 fd50:840e:74e2:320::50 fd50:840e:74e2:321::50"
YAML
# T29 asserted a string the dc1 overlay alone already produces, so its own fixture was
# INERT and the case could not fail on it (2026-07-29 chain audit; same defect class as
# T20/T21 the day before). Re-pointed at what ONLY this path proves: a dc1 v6 leg is
# validated against dc1's apex /64s, which a dc0 run cannot reach.
python3 - "$REPO" "$TMP/apex-nodc1.json" <<'PY'
import json, glob, sys
rec = sorted(glob.glob(sys.argv[1] + "/netbox/draft/vr1-office1-current-*.json"))[-1]
d = json.load(open(rec))
d["ipam/prefixes"] = [p for p in d.get("ipam/prefixes", [])
                      if (p.get("scope_site") or "") != "vr1-dc1"]
json.dump(d, open(sys.argv[2], "w"))
PY
APEX_RECORD="$TMP/apex-nodc1.json" runargs 2 'carries no v6 prefix for .* in vr1-dc1' \
  "T29 dc1 v6 legs really come from dc1's OWN apex entries (strip them -> REFUSE)" \
  "$TMP/good.yaml" --overlay "$REPO/overlays/vr1-dc1-vips.yaml" --dc vr1-dc1

# T30 the v6 analogue of T15: dc0 v6 legs under --dc vr1-dc1 must FAIL, so a
#     cross-DC copy-paste of a rendered overlay cannot pass.
cat > "$TMP/dc1-dual-wrong.yaml" <<'YAML'
applications:
  keystone:
    options:
      prefer-ipv6: true
      vip: "10.12.64.50 10.12.68.50 10.12.72.50 2602:f3e2:f02:11::50 fd50:840e:74e2:220::50 fd50:840e:74e2:221::50"
YAML
runargs 1 'provider v6 leg .* not in 2602:f3e2:f03:11::/64' \
  "T30 dc0 v6 legs under --dc vr1-dc1 FAIL (cross-DC leak)" \
  "$TMP/good.yaml" --overlay "$REPO/overlays/vr1-dc1-vips.yaml" \
  --overlay "$TMP/dc1-dual-wrong.yaml" --dc vr1-dc1

# ---- Invariant 10: HA ARITY -- cluster_count == the principal's num_units ----
# Until 2026-07-29 `cluster_count` was asserted NOWHERE in scripts/ or tests/, against
# 20 occurrences in overlays/dc-ha-scaleup.yaml, so a 3->1 rewrite of every value was a
# BYTE-IDENTICAL PASS. These cases are the proof that is over.

# T32 the GREEN twin first: the real HA overlay declares matching arity for all 13.
runargs 0 '13 hacluster subordinate\(s\) declare cluster_count == principal num_units' \
  "T32 real dc-ha-scaleup overlay: arity matches for all 13 subordinates" \
  "$TMP/good.yaml" --overlay "$REPO/overlays/dc-ha-scaleup.yaml"

# T33 THE decorative-HA fixture: every cluster_count in the shipped overlay rewritten
#     3 -> 1, num_units left at 3. This exact rewrite used to exit 0.
sed 's/cluster_count: 3/cluster_count: 1/g' "$REPO/overlays/dc-ha-scaleup.yaml" > "$TMP/ha-3to1.yaml"
runargs 1 'cluster_count=1 but principal .* num_units=3 -- DECORATIVE HA' \
  "T33 cluster_count 3->1 across the HA overlay FAILS (was byte-identical PASS)" \
  "$TMP/good.yaml" --overlay "$TMP/ha-3to1.yaml"

# T34 the OTHER direction: a cluster_count of 3 outliving a scale-down to 1 unit.
#     Equality, not a floor, is what catches this.
sed 's/^    num_units: 3/    num_units: 1/; s/^  \(.*\):$/  \1:/' "$REPO/overlays/dc-ha-scaleup.yaml" \
  | sed 's/num_units: 3/num_units: 1/g' > "$TMP/ha-scaledown.yaml"
runargs 1 'cluster_count=3 but principal .* num_units=1 -- DECORATIVE HA' \
  "T34 cluster_count 3 outliving a scale-down to 1 FAILS" \
  "$TMP/good.yaml" --overlay "$TMP/ha-scaledown.yaml"

# T35 an ABSENT cluster_count refuses rather than accepting the charm default --
#     "could not look" is never "nothing there". Mutated in the BASE, not an overlay:
#     juju's overlay merge has no delete-a-nested-key form (a null deletes a whole
#     application, not one option), so an overlay CANNOT unset cluster_count -- only
#     an edit to bundle.yaml can, which is precisely the edit this case guards.
mutate t35.yaml 'b["applications"]["keystone-hacluster"]["options"].pop("cluster_count")'
run 1 'declares NO cluster_count' \
  "T35 hacluster subordinate with no cluster_count FAILS" "$TMP/t35.yaml"

# ---- Invariant 11: MACHINES-BLOCK DC IDENTITY ----
# One MAAS region sees BOTH DCs' nodes, so the machines `tags=` is the only thing
# keeping dc1's model off dc0's hardware. All three shapes below used to exit 0.

# T36 the dc1 VIP overlay WITHOUT the machines overlay -- the exact input T14 used to
#     assert was fine. dc1 VIPs over dc0-tagged machines.
runargs 1 'machines block is tagged openstack-vr1-dc0 but --dc vr1-dc1' \
  "T36 dc1 input with the machines overlay MISSING FAILS" \
  "$TMP/good.yaml" --overlay "$REPO/overlays/vr1-dc1-vips.yaml" --dc vr1-dc1

# T37 a GUTTED machines overlay: it parses, it merges, it contributes nothing.
printf 'machines: {}\n' > "$TMP/mach-gutted.yaml"
runargs 1 'machines block is tagged openstack-vr1-dc0 but --dc vr1-dc1' \
  "T37 gutted machines overlay FAILS (parses, contributes nothing)" \
  "$TMP/good.yaml" --overlay "$TMP/mach-gutted.yaml" \
  --overlay "$REPO/overlays/vr1-dc1-vips.yaml" --dc vr1-dc1

# T38 a MIS-KEYED machines overlay ("01" where the base says "0"): the entries land
#     BESIDE the base's instead of over them, so both tag sets survive. This one is
#     caught under EITHER --dc, which T39 proves -- it needs no declared DC to see it.
sed 's/^  "\([0-8]\)":/  "0\1":/' "$REPO/overlays/vr1-dc1-machines.yaml" > "$TMP/mach-miskeyed.yaml"
runargs 1 'machines block MIXES DC tags' \
  "T38 mis-keyed machines overlay FAILS (--dc vr1-dc1)" \
  "$TMP/good.yaml" --overlay "$TMP/mach-miskeyed.yaml" \
  --overlay "$REPO/overlays/vr1-dc1-vips.yaml" --dc vr1-dc1
runargs 1 'machines block MIXES DC tags' \
  "T39 ...and under --dc vr1-dc0 too (mixing needs no declared DC)" \
  "$TMP/good.yaml" --overlay "$TMP/mach-miskeyed.yaml"

# T40 the inert-entry assertion in its own right: a machine no application places to.
mutate t40.yaml 'b["machines"]["9"]={"constraints":"arch=amd64 tags=openstack-vr1-dc0,storage"}'
run 1 'DEFINED but no application places' \
  "T40 an unreferenced machine entry FAILS" "$TMP/t40.yaml"

# T41 a machine with no DC tag at all -- allocatable from EITHER DC.
mutate t41.yaml 'b["machines"]["4"]={"constraints":"arch=amd64 tags=compute"}'
run 1 'carries no single DC tag' \
  "T41 machine with no DC tag FAILS" "$TMP/t41.yaml"

# T42 THE BYPASS these checks nearly shipped with: an overlay rewriting every machine's
#     constraints to a bare `arch=amd64` strips the role tags AND the DC tag together.
#     While invariant 11 lived inside the role-separation branch, role_sep went False and
#     the entire placement block -- the DC-tag assertions included -- self-skipped behind
#     an [ok]. So the most destructive machines overlay of all exited 0. The checks are
#     now keyed on the machines block existing, not on role tags being present.
{ printf 'machines:\n'
  for i in 0 1 2 3 4 5 6 7 8; do printf '  "%s": {constraints: arch=amd64}\n' "$i"; done
} > "$TMP/mach-untagged.yaml"
runargs 1 'carries no single DC tag' \
  "T42 overlay stripping ALL tags FAILS (does not self-skip via role_sep)" \
  "$TMP/good.yaml" --overlay "$TMP/mach-untagged.yaml" \
  --overlay "$REPO/overlays/vr1-dc1-vips.yaml" --dc vr1-dc1

echo
echo "RESULT: PASS=$PASS FAIL=$FAIL"
[[ "$FAIL" -eq 0 ]] && { echo "ALL PASS"; exit 0; } || exit 1