diff --git a/bundle.yaml b/bundle.yaml index a253cf9..cb22221 100644 --- a/bundle.yaml +++ b/bundle.yaml @@ -55,19 +55,9 @@ variables: # ----- UCA pocket + Ceph source ---------------------------------------------- openstack-origin: &openstack-origin cloud:jammy-caracal - ceph-source: &ceph-source cloud:jammy-caracal + ceph-source: &ceph-source cloud:jammy-caracal # ----- Bindings for external-API-facing charms (public on provider) ---------- - api-bindings: &api-bindings - "": metal - public: provider - - # ----- Bindings for backend / internal-only charms (all metal) --------------- - # Used for ovn-central, mysql-innodb-cluster, rabbitmq-server, vault, memcached, and the - # mysql-router subordinates. (ceph-mon / ceph-osd now use explicit public/cluster bindings, see C2.) - internal-bindings: &internal-bindings - "": metal - machines: "8": constraints: arch=amd64 tags=openstack @@ -78,6 +68,18 @@ "11": constraints: arch=amd64 tags=openstack +# ===================================================================== +# Network-space bindings (D-052): EXPLICIT per-application blocks, no anchors. +# "" -> metal-admin (operator/MAAS/monitoring; admin API; default) +# internal/shared-db/amqp/certificates/cluster/identity/ovsdb -> metal-internal +# public -> provider-public (public API + floating IPs) +# ceph public -> storage ; ceph cluster -> replication +# geneve overlay -> fabric-data (nova-compute:neutron-plugin, ovn-chassis:data, +# ovn-chassis-octavia:data, octavia:ovsdb-cms) +# Subordinate subset rule: a subordinate's spaces are a subset of its principal's; +# nova-compute keeps fabric-data (via neutron-plugin) for the ovn-chassis geneve. +# Re-IP is MAAS-side only (no CIDR options here). See docs/design-decisions.md D-052. +# ===================================================================== applications: # ===================================================================== @@ -92,7 +94,13 @@ channel: 8.0/stable num_units: 3 to: [lxd:8, lxd:9, lxd:10] - bindings: *internal-bindings + bindings: + '': metal-admin + certificates: metal-internal + cluster: metal-internal + coordinator: metal-internal + db-router: metal-internal + shared-db: metal-internal constraints: arch=amd64 rabbitmq-server: @@ -100,7 +108,12 @@ channel: 3.9/stable num_units: 1 to: [lxd:10] - bindings: *internal-bindings + bindings: + '': metal-admin + amqp: metal-internal + certificates: metal-internal + cluster: metal-internal + ha: metal-internal constraints: arch=amd64 vault: @@ -108,13 +121,24 @@ channel: 1.8/stable num_units: 1 # 3 on Roosevelt (D-009); HA backend decided there (C1) to: [lxd:11] - bindings: *internal-bindings + bindings: + '': metal-admin + access: metal-internal + certificates: metal-internal + cluster: metal-internal + ha: metal-internal + secrets: metal-internal + shared-db: metal-internal constraints: arch=amd64 vault-mysql-router: charm: mysql-router channel: 8.0/stable - bindings: *internal-bindings + bindings: + '': metal-admin + certificates: metal-internal + db-router: metal-internal + shared-db: metal-internal # ===================================================================== # Identity: Keystone @@ -126,15 +150,34 @@ num_units: 1 # 3 on Roosevelt (D-009) to: [lxd:8] options: - vip: "10.12.4.50 10.12.8.50" # B1 front-loaded VIP; IS the catalog endpoint (B5, no os-public-hostname) + vip: "10.12.4.50 10.12.8.50 10.12.12.50" # B1 front-loaded VIP; IS the catalog endpoint (B5, no os-public-hostname) use-policyd-override: true # as-built reconcile 2026-06-09 (origin untraced -- Review-later) - bindings: *api-bindings + bindings: + '': metal-admin + certificates: metal-internal + cluster: metal-internal + domain-backend: metal-internal + ha: metal-internal + identity-admin: metal-internal + identity-credentials: metal-internal + identity-notifications: metal-internal + identity-service: metal-internal + internal: metal-internal + keystone-fid-service-provider: metal-internal + keystone-middleware: metal-internal + public: provider-public + shared-db: metal-internal + websso-trusted-dashboard: metal-internal constraints: arch=amd64 keystone-mysql-router: charm: mysql-router channel: 8.0/stable - bindings: *internal-bindings + bindings: + '': metal-admin + certificates: metal-internal + db-router: metal-internal + shared-db: metal-internal # ===================================================================== # Image: Glance + simplestreams-sync @@ -146,18 +189,33 @@ num_units: 1 to: [lxd:11] options: - vip: "10.12.4.53 10.12.8.53" # B1 + vip: "10.12.4.53 10.12.8.53 10.12.12.53" # B1 image-conversion: true # as-built; image conversion enabled (raw on Ceph-backed glance) - bindings: # api-bindings + ceph->storage (C2; glance is a Ceph client) - "": metal - public: provider - ceph: storage + bindings: + '': metal-admin + amqp: metal-internal + ceph: storage + certificates: metal-internal + cinder-volume-service: metal-internal + cluster: metal-internal + ha: metal-internal + identity-service: metal-internal + image-service: metal-internal + internal: metal-internal + object-store: metal-internal + public: provider-public + shared-db: metal-internal + storage-backend: metal-internal constraints: arch=amd64 glance-mysql-router: charm: mysql-router channel: 8.0/stable - bindings: *internal-bindings + bindings: + '': metal-admin + certificates: metal-internal + db-router: metal-internal + shared-db: metal-internal glance-simplestreams-sync: charm: glance-simplestreams-sync @@ -167,7 +225,12 @@ options: # B4 amphora-pipeline use-internal-endpoints: true # use internal (IP) catalog endpoints use_swift: false # skip swift index; sidesteps radosgw object path for the amphora seed - bindings: *internal-bindings + bindings: + '': metal-admin + certificates: metal-internal + identity-service: metal-internal + image-modifier: metal-internal + simplestreams-image-service: metal-internal constraints: arch=amd64 # ===================================================================== @@ -182,8 +245,28 @@ options: console-access-protocol: novnc network-manager: Neutron - vip: "10.12.4.56 10.12.8.56" # B1 - bindings: *api-bindings + vip: "10.12.4.56 10.12.8.56 10.12.12.56" # B1 + bindings: + '': metal-admin + amqp: metal-internal + amqp-cell: metal-internal + certificates: metal-internal + cinder-volume-service: metal-internal + cloud-compute: metal-internal + cloud-controller: metal-internal + cluster: metal-internal + dashboard: metal-internal + ha: metal-internal + identity-service: metal-internal + image-service: metal-internal + internal: metal-internal + memcache: metal-internal + neutron-api: metal-internal + nova-cell-api: metal-internal + placement: metal-internal + public: provider-public + shared-db: metal-internal + shared-db-cell: metal-internal constraints: arch=amd64 nova-compute: @@ -200,17 +283,30 @@ resume-guests-state-on-host-boot: true virt-type: qemu # Testcloud nested-KVM; Roosevelt will use 'kvm' reserved-host-memory: 8192 # ENV(testcloud 16GiB hosts) D-040 OOM fix; charm default 512 -- DO NOT drop - bindings: # C2 ceph/ceph-access -> storage. OVN-on-data: neutron-plugin -> data - "": metal # puts 'data' in this principal's binding set so ovn-chassis' data - ceph: storage # binding is a valid SUBSET (subordinate subset rule). nova-compute is - ceph-access: storage # bare metal -- enp8s0 (data) is already present, so this only needs - neutron-plugin: data # to satisfy the rule, not provision a NIC. + bindings: + '': metal-admin + amqp: metal-internal + ceph: storage + ceph-access: storage + cloud-compute: metal-internal + cloud-credentials: metal-internal + compute-peer: metal-internal + image-service: metal-internal + internal: metal-internal + migration: metal-internal + neutron-plugin: fabric-data + secrets-storage: metal-internal + storage-backend: metal-internal constraints: arch=amd64 ncc-mysql-router: charm: mysql-router channel: 8.0/stable - bindings: *internal-bindings + bindings: + '': metal-admin + certificates: metal-internal + db-router: metal-internal + shared-db: metal-internal placement: charm: placement @@ -218,14 +314,28 @@ num_units: 1 to: [lxd:11] options: - vip: "10.12.4.59 10.12.8.59" # B1 - bindings: *api-bindings + vip: "10.12.4.59 10.12.8.59 10.12.12.59" # B1 + bindings: + '': metal-admin + amqp: metal-internal + certificates: metal-internal + cluster: metal-internal + ha: metal-internal + identity-service: metal-internal + internal: metal-internal + placement: metal-internal + public: provider-public + shared-db: metal-internal constraints: arch=amd64 placement-mysql-router: charm: mysql-router channel: 8.0/stable - bindings: *internal-bindings + bindings: + '': metal-admin + certificates: metal-internal + db-router: metal-internal + shared-db: metal-internal # ===================================================================== # Networking: Neutron + OVN @@ -240,25 +350,53 @@ enable-ml2-port-security: true flat-network-providers: physnet1 neutron-security-groups: true - vip: "10.12.4.55 10.12.8.55" # B1 - bindings: *api-bindings + vip: "10.12.4.55 10.12.8.55 10.12.12.55" # B1 + bindings: + '': metal-admin + amqp: metal-internal + certificates: metal-internal + cluster: metal-internal + ha: metal-internal + identity-service: metal-internal + internal: metal-internal + neutron-api: metal-internal + neutron-plugin-api: metal-internal + neutron-plugin-api-subordinate: metal-internal + public: provider-public + shared-db: metal-internal constraints: arch=amd64 neutron-api-mysql-router: charm: mysql-router channel: 8.0/stable - bindings: *internal-bindings + bindings: + '': metal-admin + certificates: metal-internal + db-router: metal-internal + shared-db: metal-internal neutron-api-plugin-ovn: charm: neutron-api-plugin-ovn channel: 2024.1/stable + bindings: + '': metal-admin + certificates: metal-internal + neutron-plugin: metal-internal + ovsdb-cms: metal-internal ovn-central: charm: ovn-central channel: 24.03/stable num_units: 3 to: [lxd:8, lxd:9, lxd:10] - bindings: *internal-bindings + bindings: + '': metal-admin + certificates: metal-internal + coordinator: metal-internal + ovsdb: metal-internal + ovsdb-cms: metal-internal + ovsdb-peer: metal-internal + ovsdb-server: metal-internal constraints: arch=amd64 # ovn-chassis: subordinate to nova-compute. MAC-based bridge-interface-mappings captured from @@ -275,24 +413,23 @@ br-ex:52:54:00:9d:63:77 br-ex:52:54:00:89:7f:ce br-ex:52:54:00:99:fc:c2 - bindings: # OVN-on-data (fidelity): geneve encap onto the data space. - "": metal # 'data' endpoint verified on the charm (was the overlay-suffix bug). - data: data # Prereq: enp8s0 link-subnet to 10.12.12.0/22 (rebuild-prep, machines Ready). - - # ovn-chassis-octavia: separate ovn-chassis app, subordinate to octavia. No bridge-interface-mappings - # and NO prefer-chassis-as-gw -- Octavia mgmt traffic rides the Neutron tenant overlay; it needs no - # external physnet bridge or gateway here. + bindings: + '': metal-admin + amqp: metal-internal + certificates: metal-internal + data: fabric-data + ovsdb: metal-internal + ovsdb-subordinate: metal-internal ovn-chassis-octavia: charm: ovn-chassis channel: 24.03/stable - bindings: # OVN-on-data: octavia chassis must share the compute chassis' - "": metal # encap network or cross-chassis geneve tunnels break. The octavia - data: data # CONTAINER gets its data NIC Juju-provisioned at deploy. - - # ===================================================================== - # Block Storage: Cinder + cinder-ceph - # ===================================================================== - + bindings: + '': metal-admin + amqp: metal-internal + certificates: metal-internal + data: fabric-data + ovsdb: metal-internal + ovsdb-subordinate: metal-internal cinder: charm: cinder channel: 2024.1/stable @@ -301,32 +438,42 @@ options: block-device: None glance-api-version: 2 - vip: "10.12.4.52 10.12.8.52" # B1 - bindings: # api-bindings + ceph -> storage. cinder's container needs a storage NIC - "": metal # for Ceph; binding the regular 'ceph' endpoint provisions it AND puts - public: provider # 'storage' in cinder's binding set, so cinder-ceph's ceph->storage is a - ceph: storage # valid subset (subset rule). cinder:ceph is unrelated here -- cinder-ceph + vip: "10.12.4.52 10.12.8.52 10.12.12.52" # B1 + bindings: + '': metal-admin + amqp: metal-internal + backup-backend: metal-internal + ceph: storage + certificates: metal-internal + cinder-volume-service: metal-internal + cluster: metal-internal + ha: metal-internal + identity-credentials: metal-internal + identity-service: metal-internal + image-service: metal-internal + internal: metal-internal + public: provider-public + shared-db: metal-internal + storage-backend: metal-internal constraints: arch=amd64 # owns the relation -- but the binding still provisions the NIC. cinder-mysql-router: charm: mysql-router channel: 8.0/stable - bindings: *internal-bindings + bindings: + '': metal-admin + certificates: metal-internal + db-router: metal-internal + shared-db: metal-internal cinder-ceph: charm: cinder-ceph channel: 2024.1/stable - bindings: # C2: Ceph client traffic -> storage. Subordinate to cinder; the principal - "": metal # (cinder:ceph -> storage) now carries 'storage', so this is a valid - ceph: storage # subset and the shared container gets a 10.12.16.x NIC. - - # ===================================================================== - # Ceph: mon + osd + radosgw (Squid release per D-005) - # ===================================================================== - # C2: full network separation via network-space BINDINGS (public/cluster) -- NOT ceph-*-network config, - # which selects the net but never provisions the LXD-contained mon a storage NIC. Hosts carry enp9s0 - # (storage) + enp10s0 (replication); clients bind ceph->storage to reach the Ceph public net. - + bindings: + '': metal-admin + ceph: storage + ceph-access: storage + storage-backend: metal-internal ceph-mon: charm: ceph-mon channel: squid/stable @@ -336,9 +483,17 @@ source: *ceph-source expected-osd-count: 4 monitor-count: 3 - bindings: # C2 -- public BINDING (NOT ceph-public-network config). The config - "": metal # selects the net but does NOT give the LXD-contained mon a storage - public: storage # NIC, so the mon can't listen on 10.12.16.0/22. The binding both + bindings: + '': metal-admin + bootstrap-source: storage + client: storage + cluster: replication + mds: storage + mon: storage + osd: storage + public: storage + radosgw: storage + rbd-mirror: storage constraints: arch=amd64 # provisions the NIC and sets the Ceph public net. Mons use only the # public net (no cluster binding needed). @@ -350,10 +505,12 @@ options: source: *ceph-source osd-devices: /dev/vdb # libvirt-attached, MAAS-untracked, wiped pre-deploy - bindings: # C2 -- public/cluster BINDINGS (NOT ceph-*-network config). Bare-metal - "": metal # OSDs already carry enp9s0 (storage) + enp10s0 (replication); the - public: storage # bindings select them as Ceph public (client) and cluster (OSD-to-OSD - cluster: replication # replication/recovery), and keep parity with ceph-mon's method. + bindings: + '': metal-admin + cluster: replication + mon: storage + public: storage + secrets-storage: metal-internal constraints: arch=amd64 tags=openstack ceph-radosgw: @@ -363,11 +520,20 @@ to: [lxd:8] options: source: *ceph-source - vip: "10.12.4.60 10.12.8.60" # B1 -- radosgw HA un-deferred for Roosevelt fidelity (decorative HA on testcloud) - bindings: # api-bindings + mon->storage (C2). radosgw IS externally-facing (S3/Swift API). - "": metal - public: provider - mon: storage + vip: "10.12.4.60 10.12.8.60 10.12.12.60" # B1 -- radosgw HA un-deferred for Roosevelt fidelity (decorative HA on testcloud) + bindings: + '': metal-admin + certificates: metal-internal + cluster: metal-internal + gateway: metal-internal + ha: metal-internal + identity-service: metal-internal + internal: metal-internal + mon: storage + object-store: metal-internal + public: provider-public + radosgw-user: metal-internal + s3: metal-internal constraints: arch=amd64 # ===================================================================== @@ -381,14 +547,31 @@ to: [lxd:10] options: debug: "false" - vip: "10.12.4.58 10.12.8.58" # B1 -- browse HTTPS by IP (B5); ALLOWED_HOSTS must permit the VIP IP (verify at deploy) - bindings: *api-bindings + vip: "10.12.4.58 10.12.8.58 10.12.12.58" # B1 -- browse HTTPS by IP (B5); ALLOWED_HOSTS must permit the VIP IP (verify at deploy) + bindings: + '': metal-admin + application-dashboard: metal-internal + certificates: metal-internal + cluster: metal-internal + dashboard: metal-internal + dashboard-plugin: metal-internal + ha: metal-internal + identity-service: metal-internal + public: provider-public + shared-db: metal-internal + website: metal-internal + websso-fid-service-provider: metal-internal + websso-trusted-dashboard: metal-internal constraints: arch=amd64 dashboard-mysql-router: charm: mysql-router channel: 8.0/stable - bindings: *internal-bindings + bindings: + '': metal-admin + certificates: metal-internal + db-router: metal-internal + shared-db: metal-internal # ===================================================================== # Load Balancer: Octavia @@ -412,22 +595,40 @@ # juju deploy ./bundle.yaml \ # --overlay overlays/vr0-dc0-testcloud.yaml \ # --overlay overlays/octavia-pki.yaml - vip: "10.12.4.57 10.12.8.57" # B1 - bindings: # api-bindings + ovsdb-cms -> data. octavia's CONTAINER needs a data NIC so - "": metal # ovn-chassis-octavia can geneve-encap on the overlay; ovsdb-cms is a - public: provider # REGULAR (octavia<->ovn-central) endpoint -- unused in the amphora-driver - ovsdb-cms: data # setup, so binding it just provisions the NIC AND makes 'data' a valid + vip: "10.12.4.57 10.12.8.57 10.12.12.57" # B1 + bindings: + '': metal-admin + amqp: metal-internal + certificates: metal-internal + cluster: metal-internal + ha: metal-internal + identity-service: metal-internal + internal: metal-internal + neutron-api: metal-internal + neutron-openvswitch: metal-internal + ovsdb-cms: fabric-data + ovsdb-subordinate: metal-internal + public: provider-public + shared-db: metal-internal constraints: arch=amd64 # subset for the subordinate's data binding (subset rule). octavia-mysql-router: charm: mysql-router channel: 8.0/stable - bindings: *internal-bindings + bindings: + '': metal-admin + certificates: metal-internal + db-router: metal-internal + shared-db: metal-internal octavia-dashboard: charm: octavia-dashboard channel: 2024.1/stable + bindings: + '': metal-admin + certificates: metal-internal + dashboard: metal-internal octavia-diskimage-retrofit: charm: octavia-diskimage-retrofit channel: 2024.1/stable @@ -442,6 +643,10 @@ # Secrets: Barbican # ===================================================================== + bindings: + '': metal-admin + certificates: metal-internal + identity-credentials: metal-internal barbican: charm: barbican channel: 2024.1/stable @@ -449,14 +654,28 @@ to: [lxd:11] options: openstack-origin: *openstack-origin - vip: "10.12.4.51 10.12.8.51" # B1 - bindings: *api-bindings + vip: "10.12.4.51 10.12.8.51 10.12.12.51" # B1 + bindings: + '': metal-admin + amqp: metal-internal + certificates: metal-internal + cluster: metal-internal + ha: metal-internal + identity-service: metal-internal + internal: metal-internal + public: provider-public + secrets: metal-internal + shared-db: metal-internal constraints: arch=amd64 barbican-mysql-router: charm: mysql-router channel: 8.0/stable - bindings: *internal-bindings + bindings: + '': metal-admin + certificates: metal-internal + db-router: metal-internal + shared-db: metal-internal barbican-vault: charm: barbican-vault @@ -475,6 +694,11 @@ # 5. magnum trustee domain-setup (REQUIRED; D-046); per-cluster app-creds are # minted by magnum at cluster-create -- NO static capo user/app-cred (D-039) + bindings: + '': metal-admin + certificates: metal-internal + secrets: metal-internal + secrets-storage: metal-internal magnum: charm: magnum channel: 2024.1/stable @@ -483,14 +707,27 @@ options: openstack-origin: *openstack-origin region: RegionOne - vip: "10.12.4.54 10.12.8.54" # B1 - bindings: *api-bindings + vip: "10.12.4.54 10.12.8.54 10.12.12.54" # B1 + bindings: + '': metal-admin + amqp: metal-internal + certificates: metal-internal + cluster: metal-internal + ha: metal-internal + identity-service: metal-internal + internal: metal-internal + public: provider-public + shared-db: metal-internal constraints: arch=amd64 magnum-mysql-router: charm: mysql-router channel: 8.0/stable - bindings: *internal-bindings + bindings: + '': metal-admin + certificates: metal-internal + db-router: metal-internal + shared-db: metal-internal magnum-dashboard: charm: magnum-dashboard @@ -505,17 +742,21 @@ # vault-hacluster stays commented (vault single-unit on mysql, C1 / BUNDLEFIX-002). # designate-hacluster stays deferred (D-019). # - keystone-hacluster: { charm: hacluster, channel: 2.4/stable, options: { cluster_count: 1 } } - glance-hacluster: { charm: hacluster, channel: 2.4/stable, options: { cluster_count: 1 } } - neutron-api-hacluster: { charm: hacluster, channel: 2.4/stable, options: { cluster_count: 1 } } - nova-cloud-controller-hacluster: { charm: hacluster, channel: 2.4/stable, options: { cluster_count: 1 } } - placement-hacluster: { charm: hacluster, channel: 2.4/stable, options: { cluster_count: 1 } } - openstack-dashboard-hacluster: { charm: hacluster, channel: 2.4/stable, options: { cluster_count: 1 } } - cinder-hacluster: { charm: hacluster, channel: 2.4/stable, options: { cluster_count: 1 } } - octavia-hacluster: { charm: hacluster, channel: 2.4/stable, options: { cluster_count: 1 } } - barbican-hacluster: { charm: hacluster, channel: 2.4/stable, options: { cluster_count: 1 } } - magnum-hacluster: { charm: hacluster, channel: 2.4/stable, options: { cluster_count: 1 } } - ceph-radosgw-hacluster: { charm: hacluster, channel: 2.4/stable, options: { cluster_count: 1 } } # B1 -- un-deferred + bindings: + '': metal-admin + certificates: metal-internal + dashboard: metal-internal + keystone-hacluster: {charm: hacluster, channel: 2.4/stable, options: {cluster_count: 1}, bindings: {'': metal-admin, ha: metal-internal, hanode: metal-internal, pacemaker-remote: metal-internal, peer-availability: metal-internal}} + glance-hacluster: {charm: hacluster, channel: 2.4/stable, options: {cluster_count: 1}, bindings: {'': metal-admin, ha: metal-internal, hanode: metal-internal, pacemaker-remote: metal-internal, peer-availability: metal-internal}} + neutron-api-hacluster: {charm: hacluster, channel: 2.4/stable, options: {cluster_count: 1}, bindings: {'': metal-admin, ha: metal-internal, hanode: metal-internal, pacemaker-remote: metal-internal, peer-availability: metal-internal}} + nova-cloud-controller-hacluster: {charm: hacluster, channel: 2.4/stable, options: {cluster_count: 1}, bindings: {'': metal-admin, ha: metal-internal, hanode: metal-internal, pacemaker-remote: metal-internal, peer-availability: metal-internal}} + placement-hacluster: {charm: hacluster, channel: 2.4/stable, options: {cluster_count: 1}, bindings: {'': metal-admin, ha: metal-internal, hanode: metal-internal, pacemaker-remote: metal-internal, peer-availability: metal-internal}} + openstack-dashboard-hacluster: {charm: hacluster, channel: 2.4/stable, options: {cluster_count: 1}, bindings: {'': metal-admin, ha: metal-internal, hanode: metal-internal, pacemaker-remote: metal-internal, peer-availability: metal-internal}} + cinder-hacluster: {charm: hacluster, channel: 2.4/stable, options: {cluster_count: 1}, bindings: {'': metal-admin, ha: metal-internal, hanode: metal-internal, pacemaker-remote: metal-internal, peer-availability: metal-internal}} + octavia-hacluster: {charm: hacluster, channel: 2.4/stable, options: {cluster_count: 1}, bindings: {'': metal-admin, ha: metal-internal, hanode: metal-internal, pacemaker-remote: metal-internal, peer-availability: metal-internal}} + barbican-hacluster: {charm: hacluster, channel: 2.4/stable, options: {cluster_count: 1}, bindings: {'': metal-admin, ha: metal-internal, hanode: metal-internal, pacemaker-remote: metal-internal, peer-availability: metal-internal}} + magnum-hacluster: {charm: hacluster, channel: 2.4/stable, options: {cluster_count: 1}, bindings: {'': metal-admin, ha: metal-internal, hanode: metal-internal, pacemaker-remote: metal-internal, peer-availability: metal-internal}} + ceph-radosgw-hacluster: {charm: hacluster, channel: 2.4/stable, options: {cluster_count: 1}, bindings: {'': metal-admin, ha: metal-internal, hanode: metal-internal, pacemaker-remote: metal-internal, peer-availability: metal-internal}} # B1 -- un-deferred # vault-hacluster: { charm: hacluster, channel: 2.4/stable } # C1: vault single-unit on mysql; HA at Roosevelt # v2-deferred (D-019): designate-hacluster: { charm: hacluster, channel: 2.4/stable } @@ -525,7 +766,10 @@ channel: latest/stable num_units: 1 to: [lxd:8] - bindings: *internal-bindings + bindings: + '': metal-admin + cache: metal-internal + cluster: metal-internal constraints: arch=amd64 relations: diff --git a/docs/design-decisions.md b/docs/design-decisions.md index ad98255..cef464c 100644 --- a/docs/design-decisions.md +++ b/docs/design-decisions.md @@ -736,3 +736,61 @@ **Options (unresolved):** (a) set `use-policyd-override=false` for v1 (the override is unused) and revisit when a real policy is needed; (b) keep true and supply an explicit, reviewed policy zip; (c) leave as-is and document the no-op. No decision made -- recorded as an open point to rule on (cf. D-043, also pending). **Related:** D-029 (Keystone SSO deferral), FINDING-1. + +## D-051: Tenant identity self-service via the SCS Domain Manager policy (resolves D-050) + +**Status:** Adopted 2026-06-23 (design; pre-redeploy). Implementation staged (`domain-manager-policy.yaml`), behavioral acceptance pending the bindings+policy redeploy (gate G3). RESOLVES D-050 via its option (b). + +**Context / why:** the commercial multi-DC end goal requires Model-2 tenant self-service: a tenant domain-admin must create/manage users, projects, and role assignments WITHIN THEIR OWN DOMAIN, without operator involvement and without escaping the domain. The plain `admin` role cannot do this: on Keystone it is architecturally NOT domain-confinable (a domain/project-scoped `admin` can escalate beyond its boundary -- upstream hard-coded limitation). This is the root cause behind the unreliable Horizon domain-admin experiments (admin-on-domain + OPENSTACK_KEYSTONE_PREFER_DOMAIN_TOKEN scope-switch breakage). + +**Decision:** implement the **SCS Domain Manager persona** (SCS scs-0302). A tenant domain-admin is a user holding the **`manager` role on their domain** (the `manager` role already exists on this cloud). Domain-confined identity self-service is granted by a **Keystone API policy override** shipped via the keystone charm's `policyd-override` resource (the mechanism `use-policyd-override=true`, already set, expects). The policy is `domain-manager-policy.yaml`, adapted VERBATIM from the canonical SCS reference (SovereignCloudStack/standards, scs-0302-w1-domain-manager-implementation-notes.md), Section A (base_* defaults) + Section B (Domain Manager extensions). + +Key policy properties (anti-escalation + fit): +- `is_domain_manager` = `role:manager`; every domain-manager grant is gated on a `token.domain.id == target.*.domain_id` match -> strictly domain-confined. +- `is_domain_managed_role` (the roles a manager may assign/revoke) = **member + load-balancer_member ONLY**. `admin` is NEVER in this list (SCS hard constraint; the anti-escalation guarantee). `load-balancer_member` is included because the Magnum apiserver-LB tenant flow requires it (cf. D-039). `reader` and `manager` deliberately NOT added (least privilege; designation authority stays operator-side -- a domain manager cannot propagate domain-manager rights). +- Every rule falls through to `or rule:base_* or admin_required`, so cloud-admin retains all authority and service-user auth is unchanged. + +**Why this approach (and NOT enforce_scope):** Charmed OpenStack deliberately does NOT implement Secure RBAC -- the keystone 2024.1/stable charm exposes NO `enforce-scope`/`enforce-new-defaults` option (verified live: both keys "not found"), and Canonical sets them False across the charms for upgrade safety. SCS scs-0302 states scope enforcement is NOT necessary for the Domain Manager persona on 2024.1 or below. So the persona is achieved purely by the policy override + `manager` role. Blast radius is LOW and bounded: the charm guarantees policy overrides alter ONLY tenant/user permissions, NEVER service users (keystone/glance/nova stay charm-managed). This is materially safer than a cloud-wide scope flip -- which is why the policy change is acceptable to combine with the explicit-bindings change in one redeploy. + +**Mechanism / rollback:** `zip overrides.zip domain-manager-policy.yaml` -> `juju attach-resource keystone policyd-override=overrides.zip` (use-policyd-override already true). Lands at `/etc/keystone/policy.d/`; keystone restarts. `juju status` shows `PO:` on success, `PO (broken):` if YAML is unparseable (atomic -- whole override discarded; this is the current empty-resource state D-050 flagged). Rollback = `juju config keystone use-policyd-override=false` (reverts to charm defaults; the Juju resource cannot be deleted but disabling is sufficient). + +**CRITICAL -- validation is YAML-only, not semantic:** the charm validates only that the override is parseable YAML; a misspelled rule key PASSES and silently no-ops. Acceptance MUST therefore be BEHAVIORAL (same "reports OK while broken" trap as D-046), never "PO: shows active." Gate G3: as a domain-scoped `manager`, create user/project + assign member/load-balancer_member within own domain PASS; assign `admin` or touch a different domain DENY; cloud-admin ops all still PASS. + +**base_* alignment (open verification, non-blocking):** the base_* rules in `domain-manager-policy.yaml` are the SCS template's upstream-recent defaults. They MUST be diffed against THIS deploy's live defaults (`oslopolicy-policy-generator --namespace keystone` on keystone/leader) and any divergent line replaced before the policy is treated as final. The file is shippable for behavioral testing as-is; the diff hardens base_* fidelity. [LIVE-READ PENDING] + +**Roosevelt / multi-DC:** the override is a single portable file replicated to every DC's keystone with zero per-DC bespoke work -- good "minimize delta to Roosevelt" properties. **Version caution (HARD):** this is the 2024.1 transitional (policy-based) implementation. The native Domain Manager persona ships in 2024.2 (Dalmatian); on any future upgrade to 2024.2+ this policy override MUST BE REMOVED (it conflicts with the native impl), except a customized `is_domain_managed_role` may be preserved. Track as an upgrade-time removal item. + +**Resolves:** D-050 (use-policyd-override=true with no zip -> now supplies the reviewed zip, option (b)). **Related:** D-046 (magnum trustee; the admin_required fallthrough preserves it -- risk drops from the enforce_scope-era HIGH to LOW), D-039 (per-cluster app-creds carrying load-balancer_member), D-029 (Keystone SSO deferral), D-044 (Horizon secure-cookie; same charm, do not disturb). **Supersedes the in-session enforce_scope "Strategy 1" framing, which was found impossible on this charm and is abandoned.** + +## D-052: Network-space inversion -- metal-internal carries all OpenStack service-to-service control + +**Status:** ADOPTED 2026-06-25 (operator-approved). AUTHORITATIVE; SUPERSEDES the three conflicting D-052 drafts (D-052-decision-text.md, D-052-CORRECTED-architecture.md, D-051-D-052-decisions.md) that attached incompatible designs to this number. Implementation pending (bundle regen + MAAS cutover); ceph network config verified (unset / binding-derived, see ceph sub-decision). + +**Context:** the as-deployed cloud collapses nearly all traffic onto a single `metal` space -- only `public`->provider, ceph `public`->storage, osd `cluster`->replication, and the geneve overlay->`data` are separated today. A prior agent's binding artifacts were corrupted (conflicting hallucinated definitions), so placement was re-derived from scratch against authoritative references, over a fresh pre-teardown endpoint inventory (`juju show-application` for all 50 apps; 446 endpoints). + +**Decision:** adopt full three-plane isolation as the per-DC template: +- `provider-public` (10.12.4.0/22): public API endpoints + floating IPs. +- `metal-admin` (10.12.8.0/22, UNTAGGED): operator / MAAS / monitoring / the OpenStack `admin` API endpoint + the `""` default binding. DC-LOCAL (never crosses DC fiber). +- `metal-internal` (10.12.12.0/22, TAGGED VLAN): ALL OpenStack service-to-service control -- internal API, shared-db (Galera), amqp (RabbitMQ/RPC), certificates/secrets (Vault), cache, cluster peers, OVN NB/SB DB (ovsdb*), identity service-to-service, and the inter-service relations. The MAY-CROSS-FIBER plane. +- `fabric-data` (10.12.16.0/22): tenant geneve overlay (nova-compute:neutron-plugin, ovn-chassis:data, ovn-chassis-octavia:data, octavia:ovsdb-cms). +- `storage` (10.12.32.0/22): Ceph public -- RBD client + mon + ceph control relations. +- `replication` (10.12.36.0/22): Ceph OSD cluster replication. + +Renames: provider->provider-public, data->fabric-data. storage/replication keep their names and RE-IP (.16->.32, .20->.36). `metal` SPLITS into metal-admin + metal-internal; metal-internal is a NEW tagged VLAN trunked to the LXD container hosts. The MAAS rename / re-IP / new-VLAN is a POST-TEARDOWN, PRE-DEPLOY step (the deploy gate, R1 risk). + +**Rationale:** RHOSP network isolation puts API + RPC + database on the Internal API network, PXE/provisioning on its own network, public API + FIPs on External, and Ceph on Storage / Storage-Management (never on Internal API). The Charmed OpenStack charm model implements this with separate public/internal/admin spaces and binds `shared-db` and `amqp` to the INTERNAL space. So the inversion (DB/MQ/internal-API on the service plane, not the management plane) is the DOCUMENTED pattern, not a deviation; moving only the `internal` endpoint while leaving shared-db/amqp/certificates/peers on the management plane would invert the intended posture. Three planes = three trust levels / three failure domains, and it maps cleanly onto multi-DC: service-to-service may extend across DC fiber, MAAS/admin stays DC-local. + +**Method:** a deterministic classifier (`classify_bindings.py`) over the live inventory (`live_bindings.json`): current provider/data/storage/replication -> rename or re-IP; current `metal` -> split by endpoint semantics; FAILS LOUD (NEEDS-REVIEW) on any unmatched endpoint rather than silently defaulting. Result: 446 endpoints classified, 0 unclassified. Distribution: metal-internal 252, metal-admin 160, provider-public 11, storage 17, replication 2, fabric-data 4. Artifacts: `target_bindings.json` (machine map) + `target_bindings_readable.txt` (audit). The bundle is regenerated MECHANICALLY from target_bindings.json (ruamel, comment-preserving), with a diff-identical verification gate vs HEAD. + +**Judgment calls (resolved):** +- HA heartbeat (corosync/pacemaker; ha/hanode/peer-availability/pacemaker-remote, 57 eps) -> metal-internal for v1 (Canonical default; rides the NICs the service units already have; ~11 DC-local rings). DEFERRED to Roosevelt: a DEDICATED, redundant (knet dual-link) heartbeat ring -- corosync's Totem protocol is latency/jitter-sensitive and production guidance keeps it off contended/bursty planes (Proxmox: storage must never share corosync's network; RHEL HA enforces interconnect latency limits). +- Ceph control relations (ceph-mon client/mon/osd/mds/radosgw/rbd-mirror/bootstrap-source, ceph-osd:mon; 8 eps) -> storage, co-located with the ceph data path (RHOSP keeps Ceph off Internal API; keeps the storage subsystem self-contained per-DC). VERIFIED 2026-06-25: `ceph-public-network` and `ceph-cluster-network` are UNSET on both ceph-mon and ceph-osd, so the charms DERIVE the ceph networks from the `public`/`cluster` extra-bindings -- the bindings ARE the source of truth (public->storage = ceph public network; cluster->replication = ceph cluster network), not just relation-NIC steering. CONSEQUENCE: the re-IP needs NO bundle CIDR changes -- it is handled entirely at the MAAS/space level (the repo bundle carries no CIDR-valued `*-network` options; subnet literals appear only in comments). CORRECTION: ceph-mon:cluster -> replication (the ceph cluster-network extra-binding, matching ceph-osd:cluster); it had been mis-grouped onto storage. ceph-osd:secrets-storage stays metal-internal (OSD->Vault dm-crypt keys, a secrets relation). +- Live migration (nova-compute:migration) -> metal-internal for v1. DEFERRED to Roosevelt: a dedicated migration plane + QEMU-native TLS for tenant-memory transit. +- nova<->OVN-chassis integration: nova-compute:neutron-plugin KEPT on fabric-data (rename of `data`, not moved); ovn-chassis:nova-compute + ovn-chassis-octavia:nova-compute -> metal-admin (subordinate glue). SUBORDINATE SUBSET RULE: a subordinate's bound spaces must be a subset of its principal's. ovn-chassis (subordinate on nova-compute) binds `data` -> fabric-data for geneve, which is only legal if nova-compute is itself bound to fabric-data -- and nova-compute's ONLY fabric-data binding is neutron-plugin. An earlier draft of this decision moved neutron-plugin to metal-internal "for isolation"; that would strip nova-compute's fabric-data NIC and make the chassis geneve binding illegal, so it was REVERTED (caught by the bundle regenerator's fail-loud + subset verification). ovn-chassis-octavia:data -> fabric-data is likewise backed by octavia:ovsdb-cms -> fabric-data. Same pattern holds for cinder-ceph: it binds ceph -> storage, and its principal cinder already carries storage via cinder:ceph -> storage, so the subset holds. +- Vault: `access` -> metal-internal (services fetch certs/secrets here); `external` -> metal-admin (operator / unseal path); unused db/etcd/lb-provider -> metal-admin. +- Subordinate glue (ovsdb-subordinate; magnum/octavia Horizon dashboard plugins) -> metal-internal (keep related control together; low-stakes). +- Unused optional endpoints (legacy neutron plugins external-dns/infoblox/midonet/vsd/neutron-lb, rgw multisite master/primary/secondary/slave, nova-vmware/quantum/ironic/vgpu/ceilometer/ephemeral, vault db/etcd/lb-provider, hsm) -> metal-admin via the `""` default, and OMITTED from the bundle rather than enumerated. Rebind if/when activated. + +**Roosevelt:** (1) dedicated, redundant corosync ring (heartbeat off the service plane); (2) dedicated live-migration plane + QEMU-native TLS; (3) revisit whether cross-DC service-to-service coordination on metal-internal warrants its own inter-DC plane. + +**Related:** D-051 (combined in the SAME redeploy -- RBAC is keystone-only / low blast radius); supersedes the three conflicting D-052 drafts; depends on the MAAS rename/re-IP/new-VLAN gate (R1); the regenerated bundle differs substantially from BOTH repo HEAD (old space names) and the WIP bundle.yaml (which was Option-1 scope: only `internal` on metal-internal). diff --git a/policies/domain-manager-policy.yaml b/policies/domain-manager-policy.yaml new file mode 100644 index 0000000..667275b --- /dev/null +++ b/policies/domain-manager-policy.yaml @@ -0,0 +1,117 @@ +# ============================================================================= +# domain-manager-policy.yaml -- Keystone policy override (SCS Domain Manager) +# Charmed OpenStack Caracal 2024.1 -- OLD-STYLE-DEFAULTS aligned +# ============================================================================= +# PROVENANCE (do not strip): +# - Domain-manager BRANCHES: adapted from the canonical SCS reference +# SovereignCloudStack/standards, scs-0302-w1-domain-manager-implementation-notes +# (the 2024.1-and-below transitional policy implementation). +# - FALLTHROUGH defaults: reproduced VERBATIM from THIS deployment's live policy +# (`oslopolicy-policy-generator --namespace keystone` on keystone/leader, +# captured 2026-06-24). The cloud runs OLD-STYLE policy (enforce_scope and +# enforce_new_defaults are forced FALSE by the Canonical charm), so each +# overridden rule reproduces the live OLD-STYLE default and PREPENDS the +# manager branch. NO new-style (system_scope:all) rules are introduced. +# +# WHY NO base_* BLOCK (improvement over the SCS template): +# This overlay does NOT redefine keystone's helper rules (cloud_admin, +# admin_and_matching_*, owner, domain_admin_for_grants, admin_on_*_filter, ...). +# Those remain keystone CODE DEFAULTS and are referenced by name. Each overridden +# rule = (manager branch) OR (the live default string, verbatim). This is +# behavior-preserving by construction for every non-manager actor, avoids copying +# helper bodies (which would risk drift), and preserves cloud_admin's baked-in +# admin domain/project IDs untouched. The `or rule:admin_required` tail from the +# SCS NEW-STYLE template is deliberately OMITTED -- the live defaults gate on +# cloud_admin (narrower than admin_required); appending admin_required would +# WIDEN access beyond the current default. +# +# DELIVERY (use-policyd-override already true): +# zip overrides.zip domain-manager-policy.yaml +# juju attach-resource keystone policyd-override=overrides.zip +# -> /etc/keystone/policy.d/domain-manager-policy.yaml ; keystone restarts. +# juju status: "PO:" = applied; "PO (broken):" = YAML unparseable (atomic discard). +# +# *** HARD WARNINGS *** +# 1. Validation is YAML-ONLY. A misspelled rule key PASSES and silently no-ops. +# Acceptance MUST be BEHAVIORAL (see ACCEPTANCE). Same trap as D-046. +# 2. 2024.1 ONLY. Native Domain Manager persona ships in 2024.2 (Dalmatian); on +# any upgrade past 2024.1 this overlay MUST be removed/reconciled (it would +# conflict with the native persona). [D-051] +# 3. THREE rules are marked [PENDING-LIVE-READ]: identity:list_users, +# identity:list_projects, identity:list_groups did NOT appear as explicit +# lines in the live dump (they fall through to "default": "rule:admin_required"). +# Their fallthrough below is the conservative "rule:admin_required" placeholder. +# CONFIRM with the short read before shipping; replace if the live default differs. +# +# THIS-DEPLOY ADAPTATIONS vs verbatim SCS: +# - is_domain_managed_role = member + load-balancer_member ONLY. admin NEVER in +# the list (anti-escalation). load-balancer_member INCLUDED (Magnum apiserver LB). +# reader/manager intentionally omitted (least privilege; tenants cannot propagate +# domain-manager rights). +# +# ACCEPTANCE (behavioral -- gate G3; never trust "PO:" alone): +# As a domain-scoped `manager` in domain D: +# PASS user/project create within D; role add member|load-balancer_member within D; +# user list / project list / group list within D; domain list; role list. +# DENY role add admin (not a managed role); any create/list in a different domain. +# As cloud admin: ALL prior admin operations still succeed (cloud_admin reproduced). +# ============================================================================= + +# --- Domain Manager rules (NEW rules defined by this overlay) --- + +# A domain manager is a user holding the `manager` role (assigned in domain scope). +"is_domain_manager": "role:manager" + +# Roles a domain manager may assign/revoke. *** admin MUST NOT appear here. *** +# member + load-balancer_member (the latter for the Magnum apiserver load balancer). +"is_domain_managed_role": "'member':%(target.role.name)s or 'load-balancer_member':%(target.role.name)s" + +# Grant-scoping helpers (NEW; self-contained token-domain checks, valid old-style). +"is_domain_user_project_grant": "token.domain.id:%(target.user.domain_id)s and token.domain.id:%(target.project.domain_id)s" +"is_domain_group_project_grant": "token.domain.id:%(target.group.domain_id)s and token.domain.id:%(target.project.domain_id)s" +"is_domain_level_user_grant": "token.domain.id:%(target.user.domain_id)s and token.domain.id:%(target.domain.id)s" +"is_domain_level_group_grant": "token.domain.id:%(target.group.domain_id)s and token.domain.id:%(target.domain.id)s" +"domain_manager_grant": "rule:is_domain_manager and (rule:is_domain_user_project_grant or rule:is_domain_group_project_grant or rule:is_domain_level_user_grant or rule:is_domain_level_group_grant)" + +# --- Domain / role discovery (manager branch + verbatim live default) --- +"identity:get_domain": "(rule:is_domain_manager and token.domain.id:%(target.domain.id)s) or rule:cloud_admin or rule:admin_and_matching_domain_id or token.project.domain.id:%(target.domain.id)s" +"identity:list_domains": "rule:is_domain_manager or rule:cloud_admin" +"identity:get_role": "(rule:is_domain_manager and rule:is_domain_managed_role) or rule:admin_required" +"identity:list_roles": "rule:is_domain_manager or rule:admin_required" + +# --- Users (manager branch + verbatim live default) --- +# [PENDING-LIVE-READ] list_users default not explicit in dump -> conservative admin_required +"identity:list_users": "(rule:is_domain_manager and token.domain.id:%(target.domain_id)s) or rule:admin_required" +"identity:get_user": "(rule:is_domain_manager and token.domain.id:%(target.user.domain_id)s) or rule:cloud_admin or rule:admin_and_matching_target_user_domain_id or rule:owner" +"identity:create_user": "(rule:is_domain_manager and token.domain.id:%(target.user.domain_id)s) or rule:cloud_admin or rule:admin_and_matching_user_domain_id" +"identity:update_user": "(rule:is_domain_manager and token.domain.id:%(target.user.domain_id)s) or rule:cloud_admin or rule:admin_and_matching_target_user_domain_id" +"identity:delete_user": "(rule:is_domain_manager and token.domain.id:%(target.user.domain_id)s) or rule:cloud_admin or rule:admin_and_matching_target_user_domain_id" + +# --- Projects (manager branch + verbatim live default) --- +# [PENDING-LIVE-READ] list_projects default not explicit in dump -> conservative admin_required +"identity:list_projects": "(rule:is_domain_manager and token.domain.id:%(target.domain_id)s) or rule:admin_required" +"identity:get_project": "(rule:is_domain_manager and token.domain.id:%(target.project.domain_id)s) or rule:cloud_admin or rule:admin_and_matching_target_project_domain_id or project_id:%(target.project.id)s" +"identity:create_project": "(rule:is_domain_manager and token.domain.id:%(target.project.domain_id)s) or rule:cloud_admin or rule:admin_and_matching_project_domain_id" +"identity:update_project": "(rule:is_domain_manager and token.domain.id:%(target.project.domain_id)s) or rule:cloud_admin or rule:admin_and_matching_target_project_domain_id" +"identity:delete_project": "(rule:is_domain_manager and token.domain.id:%(target.project.domain_id)s) or rule:cloud_admin or rule:admin_and_matching_target_project_domain_id" +"identity:list_user_projects": "(rule:is_domain_manager and token.domain.id:%(target.user.domain_id)s) or rule:owner or rule:admin_and_matching_domain_id" + +# --- Role assignments / grants (manager branch + managed-role gate + verbatim live default) --- +"identity:check_grant": "rule:domain_manager_grant or rule:cloud_admin or rule:domain_admin_for_grants or rule:project_admin_for_grants" +"identity:list_grants": "(rule:is_domain_manager and token.domain.id:%(target.user.domain_id)s) or (rule:is_domain_manager and token.domain.id:%(target.group.domain_id)s) or rule:cloud_admin or rule:domain_admin_for_list_grants or rule:project_admin_for_list_grants" +"identity:create_grant": "(rule:domain_manager_grant and rule:is_domain_managed_role) or rule:cloud_admin or rule:domain_admin_for_grants or rule:project_admin_for_grants" +"identity:revoke_grant": "(rule:domain_manager_grant and rule:is_domain_managed_role) or rule:cloud_admin or rule:domain_admin_for_grants or rule:project_admin_for_grants" +"identity:list_role_assignments": "(rule:is_domain_manager and token.domain.id:%(target.domain_id)s) or rule:cloud_admin or rule:admin_on_domain_filter or rule:admin_on_project_filter" + +# --- Groups (manager branch + verbatim live default) --- +# [PENDING-LIVE-READ] list_groups default not explicit in dump -> conservative admin_required +"identity:list_groups": "(rule:is_domain_manager and token.domain.id:%(target.group.domain_id)s) or rule:admin_required" +"identity:get_group": "(rule:is_domain_manager and token.domain.id:%(target.group.domain_id)s) or rule:cloud_admin or rule:admin_and_matching_target_group_domain_id" +"identity:create_group": "(rule:is_domain_manager and token.domain.id:%(target.group.domain_id)s) or rule:cloud_admin or rule:admin_and_matching_group_domain_id" +"identity:update_group": "(rule:is_domain_manager and token.domain.id:%(target.group.domain_id)s) or rule:cloud_admin or rule:admin_and_matching_target_group_domain_id" +"identity:delete_group": "(rule:is_domain_manager and token.domain.id:%(target.group.domain_id)s) or rule:cloud_admin or rule:admin_and_matching_target_group_domain_id" +"identity:list_groups_for_user": "(rule:is_domain_manager and token.domain.id:%(target.user.domain_id)s) or rule:owner or rule:admin_and_matching_target_user_domain_id" +"identity:list_users_in_group": "(rule:is_domain_manager and token.domain.id:%(target.group.domain_id)s) or rule:cloud_admin or rule:admin_and_matching_target_group_domain_id" +"identity:remove_user_from_group": "(rule:is_domain_manager and token.domain.id:%(target.group.domain_id)s and token.domain.id:%(target.user.domain_id)s) or rule:cloud_admin or rule:admin_and_matching_target_group_domain_id" +"identity:check_user_in_group": "(rule:is_domain_manager and token.domain.id:%(target.group.domain_id)s and token.domain.id:%(target.user.domain_id)s) or rule:cloud_admin or rule:admin_and_matching_target_group_domain_id" +"identity:add_user_to_group": "(rule:is_domain_manager and token.domain.id:%(target.group.domain_id)s and token.domain.id:%(target.user.domain_id)s) or rule:cloud_admin or rule:admin_and_matching_group_domain_id"