Newer
Older
openstack-caracal-dc-dc / netbox / sandbox-fidelity-check.py
#!/usr/bin/env python3
"""
Prove a seeded sandbox is a FAITHFUL replica of the upstream draft, plus an
expected delta. Read-only; compares two JSON dumps, touches no NetBox.

WHY THIS EXISTS. Counts and idempotency CANNOT prove fidelity. On 2026-07-13 the
seeder silently dropped the scope on 17 of 90 prefixes (it mapped only
`dcim.site`, and most of the VR1 draft is `dcim.region`-scoped) -- and every
count still matched, and re-running was still perfectly idempotent. It was caught
by luck on a spot-check that happened to print `site None`. This script is the
check that would have caught it on purpose.

It diffs FIELD BY FIELD on every shared object, so a dropped or mangled field is
a hard failure rather than an invisible one.

Usage:
    # dump upstream (on vcloud), and the sandbox (on office1-netbox) -- SAME
    # read-only dumper, pointed at each in turn:
    python3 netbox/prod-draft-dump.py --out netbox/draft/vr1-draft.json
    python3 netbox/prod-draft-dump.py --out /tmp/sandbox-dump.json   # NETBOX_URL=sandbox

    python3 netbox/sandbox-fidelity-check.py \
        --upstream netbox/draft/vr1-draft.json --sandbox /tmp/sandbox-dump.json

Exit 0 = faithful replica + exactly the expected delta. Exit 1 = the seeder is
dropping or mangling data.
"""

import argparse
import json
import sys

ap = argparse.ArgumentParser(description=__doc__.split("\n\n", 1)[0])
ap.add_argument("--upstream", required=True, help="JSON dump of the upstream draft")
ap.add_argument("--sandbox", required=True, help="JSON dump of the seeded sandbox")
args = ap.parse_args()

up = json.load(open(args.upstream))
sb = json.load(open(args.sandbox))

KEY={"ipam/rirs":"slug","ipam/roles":"slug","dcim/regions":"slug","dcim/sites":"slug",
     "ipam/aggregates":"prefix","ipam/prefixes":"prefix"}
EXPECTED_NEW={"ipam/roles":{"edge"},"dcim/sites":{"vr1-off1"},
  "ipam/prefixes":{"10.10.0.0/16","10.10.0.0/22","10.10.0.0/24","10.10.1.0/24",
                   "172.30.0.0/16","172.30.1.0/24",
                   "2602:f3e2:f01:100::/56","2602:f3e2:f01:100::/64"}}
bad=0
for ep,k in KEY.items():
    U={r[k]:r for r in up.get(ep,[])}
    S={r[k]:r for r in sb.get(ep,[])}
    missing=set(U)-set(S)
    extra=set(S)-set(U)
    exp=EXPECTED_NEW.get(ep,set())
    print(f"\n{ep}:  upstream={len(U)}  sandbox={len(S)}")
    if missing:
        bad+=1; print(f"  !! MISSING FROM SANDBOX ({len(missing)}): {sorted(missing)[:6]}")
    unexpected=extra-exp
    if unexpected:
        bad+=1; print(f"  !! UNEXPECTED EXTRA ({len(unexpected)}): {sorted(unexpected)[:6]}")
    elif extra:
        print(f"  + expected D-115 additions ({len(extra)}): {sorted(extra)}")
    # FIELD-LEVEL fidelity on the shared keys -- this is the check that counts cannot make
    drift=[]
    for key in sorted(set(U)&set(S)):
        u,s=U[key],S[key]
        for f in set(u)|set(s):
            if f.startswith("_"): continue
            if u.get(f)!=s.get(f):
                drift.append((key,f,u.get(f),s.get(f)))
    if drift:
        bad+=1
        print(f"  !! FIELD DRIFT on {len(set(d[0] for d in drift))} object(s), {len(drift)} field(s):")
        for key,f,uv,sv in drift[:12]:
            print(f"       {key:<26} {f:<14} upstream={uv!r}  sandbox={sv!r}")
    else:
        print("  field-level: IDENTICAL on every shared object")
print("\n" + "=" * 70)
if bad == 0:
    print("SANDBOX IS A FAITHFUL REPLICA + the expected D-115 delta")
    sys.exit(0)
print(f"!! {bad} fidelity problem(s) -- the seeder is dropping or mangling data")
sys.exit(1)