#!/usr/bin/env bash
# scripts/dc-dc-dr-drill.sh -- failover promote and failback demote/promote
# sequences for D-108's Glance rbd-mirror mechanism (tooling gap register
# item #5). Extracts runbooks/dc-dc-phase5-dr-failover-drill.md Step 10.2
# (FULL failover: force-promote at the surviving DC + a flagged, NOT
# auto-executed re-registration reminder) and Steps 11.2-11.4 (FULL failback:
# demote the recovering side, then -- ONLY after that -- demote the current
# primary before promoting the recovering side back).
#
# STANDARD, well-documented upstream Ceph administration (trust these): every
# `rbd mirror pool promote --force` / `rbd mirror pool demote` /
# `rbd mirror pool promote` call below is copied verbatim from the runbook,
# itself copied from upstream rbd-mirror docs -- nothing here is invented.
#
# GENUINELY REPO-SPECIFIC (needs review before trusting; none of it exists
# yet): the Glance POOL name, which DC is "down"/"recovering" vs "primary" at
# drill time (a live decision, not a fixed default), and the juju model/unit
# for each side. All REQUIRED arguments -- no invented defaults.
#
# THE SAFETY-CRITICAL PROPERTY OF THIS SCRIPT: failback's ordering --
# "demote the CURRENT PRIMARY before promoting the RECOVERING side" -- is the
# single most safety-critical property in the whole drill (runbook Step 11
# caution: promoting both, or promoting before demoting, creates two masters
# writing independently -- split-brain). This is HARD-CODED as call order in
# main() below (step_11_2, then step_11_4a demote-primary, then step_11_4b
# promote-recovering), not left to documentation or a flag an operator could
# reorder; step_11_4b is only ever invoked if step_11_4a's demote succeeded
# (apply mode) or was printed first (dry-run mode -- verifiable by grepping
# line order in the PLAN output). Step 11.3 (Cinder backup/restore
# reconciliation) is intentionally NOT implemented here -- it is a
# tenant/volume-specific manual reconciliation, out of scope for an
# rbd-mirror/radosgw mechanism script; a reminder is printed instead.
#
# Step 10.2's Glance re-registration (per-image `openstack image create`
# against the promoted RBD objects) is NEVER executed by this script, even
# under --apply -- the runbook is explicit that the exact invocation depends
# on the deployed Glance charm driver's behavior at execution time, which is
# unconfirmed. This script only prints a CLEARLY FLAGGED manual reminder
# listing the image identifiers supplied via --reregister-image.
#
# NO LIVE CEPH CLUSTER EXISTS THIS SESSION (2026-07-10, prep-only) -- never
# run for real, cannot be tonight. Default mode is a DRY-RUN plan print;
# --apply requires an additional typed confirmation (matching this repo's
# phase-00-teardown-destroy.sh --apply pattern) and executes one command at
# a time, aborting on the first failure. Its matching harness
# (tests/dc-dc-dr-drill/run-tests.sh) validates ONLY this script's own logic
# (argument parsing, guard clauses, dry-run output, the demote-before-promote
# line ORDER, and that --apply is blocked without confirmation) -- it does
# NOT and CANNOT validate real rbd-mirror promote/demote behavior.
#
# $DC gate: sources scripts/lib-net.sh and calls `lib_net_select_dc` once for
# EACH DC token this invocation touches (DOCFIX-151 convention) -- this
# deliberately makes any use of dc2 FAIL LOUD today (D-101/gap #3: DC2 has no
# assigned network literals, so no real DC2 Ceph cluster exists to target).
#
# Usage:
#   dc-dc-dr-drill.sh failover --dc <vr1-dc0|vr1-dc1> --model M --unit U --pool NAME \
#     --confirmed-down [--reregister-image ID ...] [--apply]
#   dc-dc-dr-drill.sh failback --pool NAME \
#     --recovering-dc <vr1-dc0|vr1-dc1> --recovering-model M --recovering-unit U \
#     --primary-dc <vr1-dc0|vr1-dc1>    --primary-model M    --primary-unit U \
#     [--skip-11-4] [--apply] [--no-prompt]
#   dc-dc-dr-drill.sh --help
#
# Exit: 0 ok | 1 bad/missing args | 2 juju not on PATH (apply only) |
#       3 $DC gate refused | 4 apply confirmation refused |
#       5 apply-mode command failed.  ASCII + LF.
set -uo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

usage() {
  cat <<'EOF'
usage:
  dc-dc-dr-drill.sh failover --dc vr1-dc0|vr1-dc1 --model M --unit U --pool NAME \
    --confirmed-down [--reregister-image ID ...] [--apply]
      (--dc/--model/--unit identify the DC being PROMOTED -- the surviving
      side. --confirmed-down is a required acknowledgement that runbook Step
      10.1's independent-path-plus-replication-plane split-brain check has
      already been done; this script does not perform that check itself.)

  dc-dc-dr-drill.sh failback --pool NAME \
    --recovering-dc vr1-dc0|vr1-dc1 --recovering-model M --recovering-unit U \
    --primary-dc vr1-dc0|vr1-dc1    --primary-model M    --primary-unit U \
    [--skip-11-4] [--apply] [--no-prompt]
      (recovering = the DC coming back up; primary = the DC currently
      serving as primary since the failover. Order is hard-coded: demote
      recovering (11.2) -> [11.3 reminder] -> demote primary (11.4a) ->
      promote recovering (11.4b), each gated on the previous apply-mode step
      succeeding. --skip-11-4 stops after 11.2/11.3, matching the runbook's
      own "optional, may stay on DC2 for a while" clause.)

  dc-dc-dr-drill.sh --help

Default mode is dry-run. --apply additionally requires a typed confirmation
read from /dev/tty (bypassed only by --no-prompt, for tested automation).
EOF
}

SUBCMD="${1:-}"
case "$SUBCMD" in
  -h|--help) usage; exit 0 ;;
  failover|failback) shift ;;
  "") echo "FAIL: a subcommand is required" >&2; usage >&2; exit 1 ;;
  *) echo "FAIL: unknown subcommand '$SUBCMD'" >&2; usage >&2; exit 1 ;;
esac

# shellcheck source=scripts/lib-net.sh
. "$HERE/lib-net.sh"

hdr()  { echo; echo "=== $* ==="; }
note() { echo "  $*"; }

# dc_gate_check <dc-token> -- computes gate status, does NOT exit. Prints
# nothing; sets DC_GATE_RC/DC_GATE_MSG for the caller to report and, later,
# to hard-block --apply on. A dry-run plan preview is not a mutation, so the
# gate is informational there; it is enforced (fail loud, exit 3) only right
# before any real execution below.
dc_gate_check() {
  DC_GATE_MSG="$(lib_net_select_dc "$1" 2>&1 1>/dev/null)"; DC_GATE_RC=$?
}
dc_gate_line() { # dc_gate_line <dc-token> -- formats the last dc_gate_check result
  if [ "$DC_GATE_RC" -eq 0 ]; then echo "\$DC gate: OK ($1)"; else echo "\$DC gate: FAILED for '$1' -- $DC_GATE_MSG"; fi
}
dc_gate_enforce() { # dc_gate_enforce <dc-token> -- HARD block before --apply execution
  dc_gate_check "$1"
  [ "$DC_GATE_RC" -eq 0 ] || { echo "FAIL: \$DC gate refused for '$1' -- STOP, do not mutate an unassigned DC ($DC_GATE_MSG)" >&2; exit 3; }
}

# =====================================================================
# failover: Step 10.2 -- force-promote at the surviving DC + a flagged,
# NEVER auto-executed re-registration reminder.
# =====================================================================
run_failover() {
  local DC="" MODEL="${MODEL:-openstack}" UNIT="" POOL="" APPLY=0 CONFIRMED_DOWN=0
  local -a REREG=()
  while [ $# -gt 0 ]; do
    case "$1" in
      -h|--help) usage; exit 0 ;;
      --dc) DC="${2:-}"; shift 2 ;;
      --model) MODEL="${2:-}"; shift 2 ;;
      --unit) UNIT="${2:-}"; shift 2 ;;
      --pool) POOL="${2:-}"; shift 2 ;;
      --confirmed-down) CONFIRMED_DOWN=1; shift ;;
      --reregister-image) REREG+=("${2:-}"); shift 2 ;;
      --apply) APPLY=1; shift ;;
      *) echo "FAIL: unknown argument '$1'" >&2; usage >&2; exit 1 ;;
    esac
  done

  [ -n "$DC" ] || { echo "FAIL: --dc is required (vr1-dc0|vr1-dc1 -- the DC being promoted)" >&2; exit 1; }
  case "$DC" in vr1-dc0|vr1-dc1) ;; *) echo "FAIL: --dc must be vr1-dc0 or vr1-dc1 (got '$DC'). D-119: bare dcN is RETIRED -- it was ambiguous across regions." >&2; exit 1 ;; esac
  [ -n "$UNIT" ] || { echo "FAIL: --unit is required" >&2; exit 1; }
  [ -n "$POOL" ] || { echo "FAIL: --pool is required (Glance pool name -- confirm, never assume 'glance')" >&2; exit 1; }

  dc_gate_check "$DC"

  hdr "dc-dc-dr-drill failover  dc=$DC  mode=$([ "$APPLY" -eq 1 ] && echo apply || echo dry-run)"
  note "juju model=$MODEL unit=$UNIT pool=$POOL"
  note "$(dc_gate_line "$DC")"

  hdr "PLAN"
  note "STEP 1 (Step 10.2 force-promote): juju ssh -m $MODEL $UNIT -- \"rbd mirror pool promote --force $POOL\""
  if [ "${#REREG[@]}" -gt 0 ]; then
    note "STEP 2 (Step 10.2 re-registration -- MANUAL, NEVER EXECUTED BY THIS SCRIPT):"
    for img in "${REREG[@]}"; do
      note "    CONFIRM against the deployed Glance charm driver, then run by hand: openstack --os-cloud <this-dc> image create --name <name-for-$img> ... (re-register $img)"
    done
  else
    note "STEP 2 (Step 10.2 re-registration): no --reregister-image given -- remember every image that needs re-registering into this DC's Glance is a separate, MANUAL, driver-specific step this script does not perform"
  fi

  if [ "$CONFIRMED_DOWN" -ne 1 ]; then
    echo
    echo "FAIL: --confirmed-down is required before this script will --apply -- Step 10.1's independent-path split-brain check must be done FIRST (out of scope for this script); dry-run plan above is printed regardless" >&2
    [ "$APPLY" -eq 1 ] && exit 1
  fi

  if [ "$APPLY" -ne 1 ]; then
    echo
    echo "re-run with --confirmed-down --apply to execute."
    echo "OK (dry-run)"
    exit 0
  fi

  dc_gate_enforce "$DC"
  command -v juju >/dev/null 2>&1 || { echo "FAIL: juju required on PATH for --apply" >&2; exit 2; }

  hdr "EXECUTE (apply mode)"
  echo "  DO STEP 1: juju ssh -m $MODEL $UNIT -- \"rbd mirror pool promote --force $POOL\""
  juju ssh -m "$MODEL" "$UNIT" -- "rbd mirror pool promote --force $POOL" </dev/null \
    || { echo "FAIL: force-promote failed -- STOP" >&2; exit 5; }
  echo "  STEP 2 (re-registration) NOT executed -- MANUAL, per the header comment; do it now per the plan above"
  echo "OK (apply) -- verify with: rbd mirror image status (per runbook Step 10.2/10.5 VERIFY)"
}

# =====================================================================
# failback: Steps 11.2-11.4 -- demote recovering, [11.3 reminder], then
# demote CURRENT PRIMARY before promote RECOVERING (hard-coded order).
# =====================================================================
run_failback() {
  local POOL=""
  local REC_DC="" REC_MODEL="${MODEL:-openstack}" REC_UNIT=""
  local PRI_DC="" PRI_MODEL="${MODEL:-openstack}" PRI_UNIT=""
  local APPLY=0 SKIP114=0 PROMPT=1

  while [ $# -gt 0 ]; do
    case "$1" in
      -h|--help) usage; exit 0 ;;
      --pool) POOL="${2:-}"; shift 2 ;;
      --recovering-dc) REC_DC="${2:-}"; shift 2 ;;
      --recovering-model) REC_MODEL="${2:-}"; shift 2 ;;
      --recovering-unit) REC_UNIT="${2:-}"; shift 2 ;;
      --primary-dc) PRI_DC="${2:-}"; shift 2 ;;
      --primary-model) PRI_MODEL="${2:-}"; shift 2 ;;
      --primary-unit) PRI_UNIT="${2:-}"; shift 2 ;;
      --skip-11-4) SKIP114=1; shift ;;
      --apply) APPLY=1; shift ;;
      --no-prompt) PROMPT=0; shift ;;
      *) echo "FAIL: unknown argument '$1'" >&2; usage >&2; exit 1 ;;
    esac
  done

  [ -n "$POOL" ] || { echo "FAIL: --pool is required" >&2; exit 1; }
  [ -n "$REC_DC" ] || { echo "FAIL: --recovering-dc is required (vr1-dc0|vr1-dc1)" >&2; exit 1; }
  [ -n "$PRI_DC" ] || { echo "FAIL: --primary-dc is required (vr1-dc0|vr1-dc1)" >&2; exit 1; }
  case "$REC_DC" in vr1-dc0|vr1-dc1) ;; *) echo "FAIL: --recovering-dc must be vr1-dc0 or vr1-dc1 (got '$REC_DC'). D-119: bare dcN is RETIRED." >&2; exit 1 ;; esac
  case "$PRI_DC" in vr1-dc0|vr1-dc1) ;; *) echo "FAIL: --primary-dc must be vr1-dc0 or vr1-dc1 (got '$PRI_DC'). D-119: bare dcN is RETIRED." >&2; exit 1 ;; esac
  [ "$REC_DC" != "$PRI_DC" ] || { echo "FAIL: --recovering-dc and --primary-dc must differ (got '$REC_DC' twice)" >&2; exit 1; }
  [ -n "$REC_UNIT" ] || { echo "FAIL: --recovering-unit is required" >&2; exit 1; }
  [ -n "$PRI_UNIT" ] || { echo "FAIL: --primary-unit is required" >&2; exit 1; }

  hdr "dc-dc-dr-drill failback  recovering=$REC_DC primary=$PRI_DC  mode=$([ "$APPLY" -eq 1 ] && echo apply || echo dry-run)"
  note "recovering: model=$REC_MODEL unit=$REC_UNIT"
  note "primary:    model=$PRI_MODEL unit=$PRI_UNIT"
  note "pool=$POOL"
  dc_gate_check "$REC_DC"; note "$(dc_gate_line "$REC_DC")"
  dc_gate_check "$PRI_DC"; note "$(dc_gate_line "$PRI_DC")"

  hdr "PLAN (fixed order -- do not reorder; this is the split-brain-safety mechanism itself)"
  note "STEP 11.2 (demote recovering, do NOT let it auto-resume as primary): juju ssh -m $REC_MODEL $REC_UNIT -- \"rbd mirror pool demote $POOL\""
  note "STEP 11.3 (Cinder reconciliation -- MANUAL, NOT implemented here): back up DC2-side drift, restore into the recovering DC once multisite sync catches up -- see runbook Step 11.3"
  if [ "$SKIP114" -eq 1 ]; then
    note "STEP 11.4 SKIPPED (--skip-11-4): stopping after 11.2/11.3, per the runbook's own 'optional, may stay on the current primary for a while' clause"
  else
    note "STEP 11.4a (demote CURRENT PRIMARY -- FIRST): juju ssh -m $PRI_MODEL $PRI_UNIT -- \"rbd mirror pool demote $POOL\""
    note "STEP 11.4b (promote RECOVERING -- SECOND, only after 11.4a succeeds): juju ssh -m $REC_MODEL $REC_UNIT -- \"rbd mirror pool promote $POOL\""
  fi

  if [ "$APPLY" -ne 1 ]; then
    echo
    echo "re-run with --apply to execute (typed confirmation required)."
    echo "OK (dry-run)"
    exit 0
  fi

  dc_gate_enforce "$REC_DC"
  dc_gate_enforce "$PRI_DC"
  command -v juju >/dev/null 2>&1 || { echo "FAIL: juju required on PATH for --apply" >&2; exit 2; }

  if [ "$PROMPT" -eq 1 ]; then
    [ -r /dev/tty ] || { echo "FAIL: --apply needs a terminal (or --no-prompt for tested automation)" >&2; exit 4; }
    printf 'Type FAILBACK to confirm demote-then-promote sequence on pool %s: ' "$POOL" > /dev/tty
    read -r ans < /dev/tty
    [ "$ans" = "FAILBACK" ] || { echo "FAIL: confirmation refused (got '$ans') -- nothing changed" >&2; exit 4; }
  fi

  hdr "EXECUTE (apply mode -- hard-coded order, abort on first failure)"
  echo "  DO STEP 11.2: juju ssh -m $REC_MODEL $REC_UNIT -- \"rbd mirror pool demote $POOL\""
  juju ssh -m "$REC_MODEL" "$REC_UNIT" -- "rbd mirror pool demote $POOL" </dev/null \
    || { echo "FAIL: 11.2 demote-recovering failed -- STOP (11.4 never reached)" >&2; exit 5; }
  echo "  STEP 11.3 NOT executed -- MANUAL, per the header comment; do the Cinder reconciliation now"

  if [ "$SKIP114" -eq 1 ]; then
    echo "OK (apply, stopped after 11.2/11.3 per --skip-11-4)"
    exit 0
  fi

  echo "  DO STEP 11.4a (demote CURRENT PRIMARY, first): juju ssh -m $PRI_MODEL $PRI_UNIT -- \"rbd mirror pool demote $POOL\""
  juju ssh -m "$PRI_MODEL" "$PRI_UNIT" -- "rbd mirror pool demote $POOL" </dev/null \
    || { echo "FAIL: 11.4a demote-primary failed -- STOP (11.4b will NOT run; recovering side stays demoted, no split-brain risk)" >&2; exit 5; }
  echo "  DO STEP 11.4b (promote RECOVERING, second): juju ssh -m $REC_MODEL $REC_UNIT -- \"rbd mirror pool promote $POOL\""
  juju ssh -m "$REC_MODEL" "$REC_UNIT" -- "rbd mirror pool promote $POOL" </dev/null \
    || { echo "FAIL: 11.4b promote-recovering failed -- STOP; primary is now demoted and recovering is NOT yet promoted, resolve by hand before retrying" >&2; exit 5; }

  echo "OK (apply) -- verify with: rbd mirror pool status --verbose + radosgw-admin sync status both zones (per runbook Step 11.5 VERIFY)"
}

case "$SUBCMD" in
  failover) run_failover "$@" ;;
  failback) run_failback "$@" ;;
esac
