diff --git a/.github/buildomat/jobs/test-ddm-quartet.sh b/.github/buildomat/jobs/test-ddm-quartet.sh index 6a4ae1373..0f2b85e65 100755 --- a/.github/buildomat/jobs/test-ddm-quartet.sh +++ b/.github/buildomat/jobs/test-ddm-quartet.sh @@ -2,7 +2,7 @@ #: #: name = "test-ddm-quartet" #: variety = "basic" -#: target = "helios-2.0" +#: target = "helios-3.0" #: rust_toolchain = "stable" #: output_rules = [ #: "/work/*.log", diff --git a/.github/buildomat/jobs/test-ddm-trio.sh b/.github/buildomat/jobs/test-ddm-trio.sh index 487b5162f..ab668eca2 100755 --- a/.github/buildomat/jobs/test-ddm-trio.sh +++ b/.github/buildomat/jobs/test-ddm-trio.sh @@ -2,7 +2,7 @@ #: #: name = "test-ddm-trio" #: variety = "basic" -#: target = "helios-2.0" +#: target = "helios-3.0" #: rust_toolchain = "stable" #: output_rules = [ #: "/work/*.log", diff --git a/.github/buildomat/test-ddm-common.sh b/.github/buildomat/test-ddm-common.sh index aefbf0278..96170c21e 100755 --- a/.github/buildomat/test-ddm-common.sh +++ b/.github/buildomat/test-ddm-common.sh @@ -1,9 +1,9 @@ #!/bin/bash export MAGHEMITE_VERSION=`git rev-parse HEAD` -export SOFTNPU_VERSION=591c64bf9765b6ed7cd8615ceb8cf6f8d117bd28 -export SIDECAR_LITE_VERSION=a95b7a9f78c08125f4e34106f5c885c7e9f2e8d5 -export DENDRITE_VERSION=72461d3a6e4724fd33454836d3c9d93c393fd4e4 +export SOFTNPU_VERSION=284c6830722548714128e63ea04bcca78ee27154 +export SIDECAR_LITE_VERSION=6f3311e8acd7e7e95c167aab61188355a93afe72 +export DENDRITE_VERSION=0d48b5a39dc49d30b228ae01f04b8cc27a161e74 function cleanup { pfexec chown -R `id -un`:`id -gn` . diff --git a/Cargo.lock b/Cargo.lock index bd1a9b40e..8fc8405bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -829,6 +829,10 @@ version = "0.1.0" dependencies = [ "omicron-common", "oxnet", + "schemars 0.8.22", + "serde", + "serde_json", + "thiserror 2.0.19", ] [[package]] @@ -901,7 +905,7 @@ dependencies = [ [[package]] name = "common" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/dendrite?rev=04a52a3240a26c58f03e1cf40353d67ff69d76b6#04a52a3240a26c58f03e1cf40353d67ff69d76b6" +source = "git+https://github.com/oxidecomputer/dendrite?rev=0d48b5a39dc49d30b228ae01f04b8cc27a161e74#0d48b5a39dc49d30b228ae01f04b8cc27a161e74" dependencies = [ "anyhow", "chrono", @@ -1378,10 +1382,12 @@ dependencies = [ "chrono", "ddm-api", "ddm-api-types", + "ddm-api-types-versions", "ddm-protocol", "dpd-client", "dropshot", "expectorate", + "futures", "hostname 0.4.2", "http-body-util", "hyper", @@ -1397,6 +1403,7 @@ dependencies = [ "oxnet", "port-file", "pretty_assertions", + "reqwest 0.13.4", "schemars 0.8.22", "serde", "serde_json", @@ -1404,6 +1411,7 @@ dependencies = [ "slog", "slog-error-chain", "socket2", + "tempfile", "thiserror 2.0.19", "tokio", "uuid", @@ -1443,11 +1451,14 @@ dependencies = [ name = "ddm-api-types-versions" version = "0.1.0" dependencies = [ + "client-common", "ddm-protocol", "oxnet", "schemars 0.8.22", "serde", + "serde_json", "serde_repr", + "thiserror 2.0.19", "uuid", ] @@ -1678,7 +1689,7 @@ dependencies = [ [[package]] name = "dpd-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/dendrite?rev=04a52a3240a26c58f03e1cf40353d67ff69d76b6#04a52a3240a26c58f03e1cf40353d67ff69d76b6" +source = "git+https://github.com/oxidecomputer/dendrite?rev=0d48b5a39dc49d30b228ae01f04b8cc27a161e74#0d48b5a39dc49d30b228ae01f04b8cc27a161e74" dependencies = [ "async-trait", "chrono", @@ -3911,13 +3922,16 @@ dependencies = [ "backoff", "clap", "ddm-api-types", + "dpd-client", "libc", "libnet", "oximeter", "oximeter-producer", "oxnet", + "proptest", "schemars 0.8.22", "serde", + "serde_json", "slog", "slog-async", "slog-bunyan", diff --git a/Cargo.toml b/Cargo.toml index 6a6d2063a..f847fce2f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -97,6 +97,7 @@ dropshot-api-manager-types = "0.7.2" expectorate = "1.2.0" schemars = { version = "0.8.22", features = [ "uuid1", "chrono" ] } tokio = { version = "1.52.1", features = ["full"] } +futures = "0.3" serde_repr = "0.1" anyhow = "1.0.104" port-file = "0.1.0" @@ -159,4 +160,4 @@ rev = "aa4714289cbf28010a821c49d9b73ec94074fb91" [workspace.dependencies.dpd-client] git = "https://github.com/oxidecomputer/dendrite" -rev = "04a52a3240a26c58f03e1cf40353d67ff69d76b6" +rev = "0d48b5a39dc49d30b228ae01f04b8cc27a161e74" diff --git a/client-common/Cargo.toml b/client-common/Cargo.toml index 545ef8cb1..272287858 100644 --- a/client-common/Cargo.toml +++ b/client-common/Cargo.toml @@ -5,6 +5,10 @@ edition = "2024" [dependencies] oxnet.workspace = true +schemars.workspace = true +serde.workspace = true +thiserror.workspace = true [dev-dependencies] omicron-common.workspace = true +serde_json.workspace = true diff --git a/client-common/src/address.rs b/client-common/src/address.rs index f5881a117..38ee24755 100644 --- a/client-common/src/address.rs +++ b/client-common/src/address.rs @@ -2,8 +2,6 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -// Copyright 2026 Oxide Computer Company - //! Multicast addressing constants shared across the routing suite. //! //! These mirror the canonical definitions in `omicron_common::address`. diff --git a/client-common/src/lib.rs b/client-common/src/lib.rs index f653328c9..601b08f92 100644 --- a/client-common/src/lib.rs +++ b/client-common/src/lib.rs @@ -3,6 +3,8 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. pub mod address; +pub mod multicast; +pub mod vni; /// Like `println!`, but silently exits on broken pipe (EPIPE) instead of /// panicking. Other I/O errors still panic. diff --git a/client-common/src/multicast.rs b/client-common/src/multicast.rs new file mode 100644 index 000000000..b42e4d785 --- /dev/null +++ b/client-common/src/multicast.rs @@ -0,0 +1,236 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Validated underlay multicast address shared across the routing suite. +//! +//! Lives in the cycle-free leaf crate so the API-types crates consumed by +//! Omicron can share a single definition without depending on +//! `omicron_common`, which would form a dependency cycle. + +use crate::address::UNDERLAY_MULTICAST_SUBNET; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::fmt; +use std::net::{IpAddr, Ipv6Addr}; +use std::str::FromStr; +use thiserror::Error; + +/// Error constructing an [`UnderlayMulticastIpv6`] address. +#[derive(Debug, Clone, Error)] +pub enum UnderlayMulticastError { + /// The address is not within the underlay multicast subnet (ff04::/64). + #[error( + "underlay address {addr} is not within {UNDERLAY_MULTICAST_SUBNET}" + )] + NotInSubnet { addr: Ipv6Addr }, + + /// The string could not be parsed as an IPv6 address. + #[error("invalid IPv6 address: {0}")] + InvalidIpv6(#[from] std::net::AddrParseError), +} + +/// Error constructing an [`OverlayMulticast`] address. +#[derive(Debug, Clone, Error)] +pub enum OverlayMulticastError { + /// The address is not a multicast address. + #[error("overlay address {addr} is not a multicast address")] + NotMulticast { addr: IpAddr }, + + /// The string could not be parsed as an IP address. + #[error("invalid IP address: {0}")] + InvalidIp(#[from] std::net::AddrParseError), +} + +/// A validated overlay multicast group address (IPv4 or IPv6). +/// +/// The application-visible group an operator announces via DDM, e.g. +/// `233.252.0.1` or `ff0e::1`. The overlay group spans both address +/// families, so this type wraps [`IpAddr`] and enforces only that the +/// address is multicast: IPv4 `224.0.0.0/4` per [RFC 1112 §4] or IPv6 +/// `ff00::/8` per [RFC 4291 §2.7]. Its admin-local underlay mapping is the +/// separately validated [`UnderlayMulticastIpv6`]. The mapping is defined by +/// [RFD 488]. +/// +/// [RFC 1112 §4]: https://www.rfc-editor.org/rfc/rfc1112#section-4 +/// [RFC 4291 §2.7]: https://www.rfc-editor.org/rfc/rfc4291#section-2.7 +/// [RFD 488]: https://rfd.shared.oxide.computer/rfd/488 +#[derive( + Debug, + Copy, + Clone, + Eq, + PartialEq, + PartialOrd, + Ord, + Hash, + Serialize, + Deserialize, + JsonSchema, +)] +#[serde(try_from = "IpAddr", into = "IpAddr")] +#[schemars(transparent)] +pub struct OverlayMulticast(IpAddr); + +impl OverlayMulticast { + /// Create a new validated overlay multicast address. + /// + /// # Errors + /// + /// Returns [`OverlayMulticastError::NotMulticast`] if the address is not + /// a multicast address. + pub fn new(value: IpAddr) -> Result { + if !value.is_multicast() { + return Err(OverlayMulticastError::NotMulticast { addr: value }); + } + Ok(Self(value)) + } + + /// Return the underlying IP address. + #[inline] + pub const fn ip(&self) -> IpAddr { + self.0 + } +} + +impl fmt::Display for OverlayMulticast { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +impl TryFrom for OverlayMulticast { + type Error = OverlayMulticastError; + + fn try_from(value: IpAddr) -> Result { + Self::new(value) + } +} + +impl From for IpAddr { + fn from(addr: OverlayMulticast) -> Self { + addr.0 + } +} + +impl FromStr for OverlayMulticast { + type Err = OverlayMulticastError; + + fn from_str(s: &str) -> Result { + let addr: IpAddr = s.parse()?; + Self::new(addr) + } +} + +/// A validated underlay multicast IPv6 address within `ff04::/64`. +/// +/// The Oxide rack maps overlay multicast groups 1:1 to admin-local scoped +/// IPv6 multicast addresses in `UNDERLAY_MULTICAST_SUBNET` (`ff04::/64`, +/// admin-local scope per [RFC 4291 §2.7]). The mapping is defined by +/// [RFD 488]. This type enforces the subnet invariant at construction +/// time. +/// +/// [RFC 4291 §2.7]: https://www.rfc-editor.org/rfc/rfc4291#section-2.7 +/// [RFD 488]: https://rfd.shared.oxide.computer/rfd/488 +// TODO: `dpd_types::mcast::UnderlayMulticastIpv6` in dendrite carries an +// independent copy. Consolidate into `oxnet`, the cycle-free leaf crate that +// maghemite, dendrite, and omicron already share, so the duplication can be +// removed. +#[derive( + Debug, + Copy, + Clone, + Eq, + PartialEq, + PartialOrd, + Ord, + Hash, + Serialize, + Deserialize, + JsonSchema, +)] +#[serde(try_from = "Ipv6Addr", into = "Ipv6Addr")] +#[schemars(transparent)] +pub struct UnderlayMulticastIpv6(Ipv6Addr); + +impl UnderlayMulticastIpv6 { + /// Create a new validated underlay multicast address. + /// + /// # Errors + /// + /// Returns [`UnderlayMulticastError::NotInSubnet`] if the address is + /// not within `UNDERLAY_MULTICAST_SUBNET` (ff04::/64). + pub fn new(value: Ipv6Addr) -> Result { + if !UNDERLAY_MULTICAST_SUBNET.contains(value) { + return Err(UnderlayMulticastError::NotInSubnet { addr: value }); + } + Ok(Self(value)) + } + + /// Return the underlying IPv6 address. + #[inline] + pub const fn ip(&self) -> Ipv6Addr { + self.0 + } +} + +impl fmt::Display for UnderlayMulticastIpv6 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +impl TryFrom for UnderlayMulticastIpv6 { + type Error = UnderlayMulticastError; + + fn try_from(value: Ipv6Addr) -> Result { + Self::new(value) + } +} + +impl From for Ipv6Addr { + fn from(addr: UnderlayMulticastIpv6) -> Self { + addr.0 + } +} + +impl From for IpAddr { + fn from(addr: UnderlayMulticastIpv6) -> Self { + IpAddr::V6(addr.0) + } +} + +impl FromStr for UnderlayMulticastIpv6 { + type Err = UnderlayMulticastError; + + fn from_str(s: &str) -> Result { + let addr: Ipv6Addr = s.parse()?; + Self::new(addr) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn overlay_serde_rejects_unicast() { + let json = + serde_json::to_string(&"192.0.2.1".parse::().unwrap()) + .unwrap(); + let result: Result = serde_json::from_str(&json); + assert!(result.is_err()); + } + + #[test] + fn underlay_serde_rejects_invalid() { + // ff0e::1 serialized as an Ipv6Addr, then deserialized as + // UnderlayMulticastIpv6 should fail via try_from. + let json = + serde_json::to_string(&Ipv6Addr::new(0xff0e, 0, 0, 0, 0, 0, 0, 1)) + .unwrap(); + let result: Result = + serde_json::from_str(&json); + assert!(result.is_err()); + } +} diff --git a/client-common/src/vni.rs b/client-common/src/vni.rs new file mode 100644 index 000000000..cc2aa18e5 --- /dev/null +++ b/client-common/src/vni.rs @@ -0,0 +1,139 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Geneve Virtual Network Identifier (VNI). +//! +//! Lives in the cycle-free leaf crate so the API-types crates consumed by +//! Omicron can share a single definition without depending on +//! `omicron_common`, which would form a dependency cycle. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::fmt::{self, Formatter}; + +/// Maximum Geneve VNI value. +/// +/// Virtual Network Identifiers are constrained to 24-bit values per the +/// Geneve specification (RFC 8926 Section 3.3). +pub const MAX_VNI: u32 = 0xFF_FFFF; + +/// Default VNI for fleet-wide multicast routing. +/// +/// A low-numbered VNI chosen to avoid colliding with user VNIs, though it is +/// not yet within the Oxide-reserved range. +pub const DEFAULT_MULTICAST_VNI: u32 = 77; + +/// Error raised while validating a [`Vni`]. +#[derive(thiserror::Error, Debug)] +pub enum VniError { + /// The value exceeds the 24-bit Geneve maximum. + #[error("VNI {value} exceeds the maximum 24-bit value {MAX_VNI}")] + OutOfRange { value: u32 }, +} + +/// A validated Geneve Virtual Network Identifier. +/// +/// Wraps a 24-bit VNI, rejecting any value above [`MAX_VNI`] at construction +/// and deserialization so an out-of-range identifier is unrepresentable. +#[derive( + Debug, + Copy, + Clone, + Eq, + PartialEq, + PartialOrd, + Ord, + Hash, + Serialize, + Deserialize, + JsonSchema, +)] +#[serde(try_from = "u32", into = "u32")] +#[schemars(transparent)] +pub struct Vni(u32); + +impl Vni { + /// Default VNI for fleet-wide multicast routing. + pub const DEFAULT_MULTICAST: Self = Self(DEFAULT_MULTICAST_VNI); + + /// Create a validated VNI. + /// + /// # Errors + /// + /// Returns [`VniError::OutOfRange`] if `value` exceeds [`MAX_VNI`], the + /// largest 24-bit Geneve VNI. + /// + /// # Examples + /// + /// ``` + /// use client_common::vni::{MAX_VNI, Vni}; + /// + /// assert!(Vni::new(77).is_ok()); + /// assert!(Vni::new(MAX_VNI + 1).is_err()); + /// ``` + pub fn new(value: u32) -> Result { + if value > MAX_VNI { + return Err(VniError::OutOfRange { value }); + } + Ok(Self(value)) + } + + /// Return the underlying 24-bit value. + #[inline] + pub const fn as_u32(self) -> u32 { + self.0 + } +} + +impl fmt::Display for Vni { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +impl TryFrom for Vni { + type Error = VniError; + + fn try_from(value: u32) -> Result { + Self::new(value) + } +} + +impl From for u32 { + fn from(vni: Vni) -> Self { + vni.0 + } +} + +#[cfg(test)] +mod tests { + use omicron_common::api::external::Vni as CanonicalVni; + + use super::*; + + /// Assert the locally copied VNI literals equal their + /// `omicron_common::api::external::Vni` originals so they cannot drift. + /// + /// `omicron_common` is a dev-dependency only, so it does not appear in the + /// normal dependency tree the no-omicron CI check inspects. + #[test] + fn vni_constants_match_canonical_values() { + assert_eq!(MAX_VNI, CanonicalVni::MAX_VNI); + assert_eq!( + DEFAULT_MULTICAST_VNI, + CanonicalVni::DEFAULT_MULTICAST_VNI.as_u32() + ); + } + + /// The [`Vni`] newtype accepts in-range values and rejects values above + /// [`MAX_VNI`], enforcing the 24-bit invariant at construction. + #[test] + fn vni_rejects_out_of_range() { + assert_eq!(Vni::new(0).unwrap().as_u32(), 0); + assert_eq!(Vni::new(MAX_VNI).unwrap().as_u32(), MAX_VNI); + assert_eq!(Vni::DEFAULT_MULTICAST.as_u32(), DEFAULT_MULTICAST_VNI); + assert!(Vni::new(MAX_VNI + 1).is_err()); + assert!(Vni::new(u32::MAX).is_err()); + } +} diff --git a/ddm-admin-client/src/lib.rs b/ddm-admin-client/src/lib.rs index 77988121c..7159b21ec 100644 --- a/ddm-admin-client/src/lib.rs +++ b/ddm-admin-client/src/lib.rs @@ -20,6 +20,12 @@ progenitor::generate_api!( }), replace = { TunnelOrigin = ddm_api_types_versions::latest::net::TunnelOrigin, + MulticastOrigin = ddm_api_types_versions::latest::net::MulticastOrigin, + UnderlayMulticastIpv6 = ddm_api_types_versions::latest::net::UnderlayMulticastIpv6, + Vni = ddm_api_types_versions::latest::net::Vni, + MulticastRoute = ddm_api_types_versions::latest::db::MulticastRoute, + MulticastPathHop = ddm_api_types_versions::latest::exchange::MulticastPathHop, + MulticastPathVector = ddm_api_types_versions::latest::exchange::MulticastPathVector, PeerInfo = ddm_api_types_versions::latest::db::PeerInfo, PeerStatus = ddm_api_types_versions::latest::db::PeerStatus, Duration = std::time::Duration, diff --git a/ddm-api-types/versions/Cargo.toml b/ddm-api-types/versions/Cargo.toml index eac3d8d3f..9ff5e7ddd 100644 --- a/ddm-api-types/versions/Cargo.toml +++ b/ddm-api-types/versions/Cargo.toml @@ -4,9 +4,12 @@ version = "0.1.0" edition = "2024" [dependencies] +client-common.workspace = true +ddm-protocol.workspace = true oxnet.workspace = true schemars.workspace = true serde.workspace = true +serde_json.workspace = true serde_repr.workspace = true +thiserror.workspace = true uuid.workspace = true -ddm-protocol.workspace = true diff --git a/ddm-api-types/versions/src/latest.rs b/ddm-api-types/versions/src/latest.rs index a2c1b47f0..0eaade4fe 100644 --- a/ddm-api-types/versions/src/latest.rs +++ b/ddm-api-types/versions/src/latest.rs @@ -13,15 +13,21 @@ pub mod admin { pub mod db { pub use crate::v1::db::RouterKind; pub use crate::v1::db::TunnelRoute; - - pub use crate::v2::db::PeerInfo; pub use crate::v2::db::PeerStatus; + pub use crate::v3::db::MulticastRoute; + pub use crate::v3::db::PeerInfo; } pub mod exchange { pub use crate::v1::exchange::PathVector; + pub use crate::v3::exchange::MulticastPathHop; + pub use crate::v3::exchange::MulticastPathVector; } pub mod net { pub use crate::v1::net::TunnelOrigin; + pub use crate::v3::net::MulticastOrigin; + pub use crate::v3::net::OverlayMulticast; + pub use crate::v3::net::UnderlayMulticastIpv6; + pub use crate::v3::net::Vni; } diff --git a/ddm-api-types/versions/src/lib.rs b/ddm-api-types/versions/src/lib.rs index 7d88b9274..bd9be63b5 100644 --- a/ddm-api-types/versions/src/lib.rs +++ b/ddm-api-types/versions/src/lib.rs @@ -34,3 +34,5 @@ pub mod latest; pub mod v1; #[path = "peer_durations/mod.rs"] pub mod v2; +#[path = "multicast_support/mod.rs"] +pub mod v3; diff --git a/ddm-api-types/versions/src/multicast_support/db.rs b/ddm-api-types/versions/src/multicast_support/db.rs new file mode 100644 index 000000000..4f21e3b39 --- /dev/null +++ b/ddm-api-types/versions/src/multicast_support/db.rs @@ -0,0 +1,161 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Database types changed in API version 3 (MULTICAST_SUPPORT). +//! +//! Adds `MulticastRoute` for routes learned via DDM and extends +//! `PeerInfo` with an optional discovery interface name. + +use std::net::Ipv6Addr; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use ddm_protocol::v4::MulticastPathHop; + +use super::net::MulticastOrigin; +use crate::v1::db::RouterKind; +use crate::v2::db::PeerStatus; + +/// A multicast route learned via DDM. +/// +/// Carries a `MulticastOrigin` (overlay group + ff04::/64 underlay +/// mapping) and the path vector from the originating subscriber +/// through intermediate transit routers. +// The path enables loop detection and (in multi-rack topologies) +// replication optimizations (RFD 488) in the future. +// +// Equality and hashing consider only `origin` and `nexthop`, so the path +// is not part of a route's identity. Updating a stored route's path +// therefore requires replacing the existing entry (e.g. `HashSet::replace`). +// `HashSet::insert` leaves the stored path unchanged. +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct MulticastRoute { + /// The multicast group origin information. + pub origin: MulticastOrigin, + + /// Underlay nexthop address (DDM peer that advertised this route). + /// Used to associate the route with a peer for expiration. + pub nexthop: Ipv6Addr, + + /// Path vector from the originating subscriber outward. + /// Each hop records the router that redistributed this + /// subscription announcement. Used for loop detection on pull + /// and for future replication optimization in multi-rack + /// topologies. + #[serde(default)] + pub path: Vec, +} + +impl MulticastRoute { + /// Identity used for equality and hashing: which group, from which peer. + /// Excludes `path`, so equality and hashing both key on the same fields + /// and cannot drift as the struct grows. + fn identity(&self) -> (&MulticastOrigin, &Ipv6Addr) { + (&self.origin, &self.nexthop) + } +} + +impl PartialEq for MulticastRoute { + fn eq(&self, other: &Self) -> bool { + self.identity() == other.identity() + } +} + +impl Eq for MulticastRoute {} + +impl std::hash::Hash for MulticastRoute { + fn hash(&self, state: &mut H) { + self.identity().hash(state); + } +} + +impl From for MulticastOrigin { + fn from(x: MulticastRoute) -> Self { + x.origin + } +} + +/// Peer information with an optional interface name. +/// +// Adds the `if_name` field to identify which underlay interface the peer +// was discovered on. +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq)] +pub struct PeerInfo { + pub status: PeerStatus, + pub addr: Ipv6Addr, + pub host: String, + pub kind: RouterKind, + /// Interface name the peer was discovered on (e.g., "tfportrear0_0"). + #[serde(default)] + pub if_name: Option, +} + +/// Downconvert v3 `PeerInfo` to v2 `PeerInfo` by dropping `if_name`. +impl From for crate::v2::db::PeerInfo { + fn from(p: PeerInfo) -> Self { + Self { + status: p.status, + addr: p.addr, + host: p.host, + kind: p.kind, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn origin(overlay: &str) -> MulticastOrigin { + serde_json::from_value(serde_json::json!({ + "overlay_group": overlay, + "underlay_group": "ff04::1", + "vni": 77, + "metric": 0 + })) + .unwrap() + } + + // The path is excluded from a route's identity, so two routes sharing an + // origin and nexthop but carrying different paths are equal. + #[test] + fn route_identity_excludes_path() { + let base = MulticastRoute { + origin: origin("233.252.0.1"), + nexthop: Ipv6Addr::LOCALHOST, + path: vec![MulticastPathHop::new( + "router-1".into(), + Ipv6Addr::LOCALHOST, + )], + }; + let mut other = base.clone(); + other.path = vec![ + MulticastPathHop::new("router-1".into(), Ipv6Addr::LOCALHOST), + MulticastPathHop::new("router-2".into(), Ipv6Addr::LOCALHOST), + ]; + assert_eq!(base, other); + } + + #[test] + fn route_identity_keys_on_origin_and_nexthop() { + let base = MulticastRoute { + origin: origin("233.252.0.1"), + nexthop: Ipv6Addr::LOCALHOST, + path: vec![], + }; + let other_origin = MulticastRoute { + origin: origin("233.252.0.2"), + nexthop: Ipv6Addr::LOCALHOST, + path: vec![], + }; + let other_nexthop = MulticastRoute { + origin: origin("233.252.0.1"), + nexthop: Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 1), + path: vec![], + }; + assert_ne!(base, other_origin); + assert_ne!(base, other_nexthop); + } +} diff --git a/ddm-api-types/versions/src/multicast_support/exchange.rs b/ddm-api-types/versions/src/multicast_support/exchange.rs new file mode 100644 index 000000000..59359b8e6 --- /dev/null +++ b/ddm-api-types/versions/src/multicast_support/exchange.rs @@ -0,0 +1,14 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Exchange (wire) types added in API version 3 (MULTICAST_SUPPORT), +//! which carries the `ddm_protocol::v4` wire types. +//! +//! These are re-exports of the plain wire types defined in the +//! [`ddm_protocol`] crate, mirroring how `PathVector` and `TunnelOrigin` +//! re-export their `ddm_protocol::v3` counterparts. Keeping a single +//! definition avoids a rich/wire split for path vectors, and peer-supplied +//! routes are stored in this plain form without re-validation. + +pub use ddm_protocol::v4::{MulticastPathHop, MulticastPathVector}; diff --git a/ddm-api-types/versions/src/multicast_support/mod.rs b/ddm-api-types/versions/src/multicast_support/mod.rs new file mode 100644 index 000000000..bb95299fb --- /dev/null +++ b/ddm-api-types/versions/src/multicast_support/mod.rs @@ -0,0 +1,12 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Types added in API version 3 (MULTICAST_SUPPORT). +//! +//! Adds multicast group origination, route distribution, and per-peer +//! discovery interface name tracking. + +pub mod db; +pub mod exchange; +pub mod net; diff --git a/ddm-api-types/versions/src/multicast_support/net.rs b/ddm-api-types/versions/src/multicast_support/net.rs new file mode 100644 index 000000000..aa0820be6 --- /dev/null +++ b/ddm-api-types/versions/src/multicast_support/net.rs @@ -0,0 +1,227 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Multicast origin and validated underlay address types added in +//! API version 3 (MULTICAST_SUPPORT). + +pub use client_common::multicast::{ + OverlayMulticast, OverlayMulticastError, UnderlayMulticastError, + UnderlayMulticastIpv6, +}; +pub use client_common::vni::{Vni, VniError}; +use ddm_protocol::v4; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::net::IpAddr; + +/// Error promoting a wire [`v4::MulticastOrigin`] to the validated +/// [`MulticastOrigin`]. +#[derive(Debug, thiserror::Error)] +pub enum MulticastOriginError { + /// The overlay group is not a multicast address. + #[error(transparent)] + OverlayGroup(#[from] OverlayMulticastError), + + /// The underlay group is not within ff04::/64. + #[error(transparent)] + UnderlayGroup(#[from] UnderlayMulticastError), + + /// The VNI exceeds the 24-bit Geneve maximum. + #[error(transparent)] + Vni(#[from] VniError), +} + +fn default_multicast_vni() -> Vni { + Vni::DEFAULT_MULTICAST +} + +/// Origin information for a multicast group announcement. +/// +/// Analogous to `TunnelOrigin` but for multicast groups. Represents a +/// subscription to a multicast group that should be advertised via DDM. +/// `overlay_group` is the application-visible multicast address (e.g., +/// 233.252.0.1 or ff0e::1), while `underlay_group` is the mapped +/// admin-local scoped IPv6 address (ff04::X) used in the underlay network. +#[derive(Debug, Clone, Eq, Serialize, Deserialize, JsonSchema)] +pub struct MulticastOrigin { + /// The overlay multicast group address (IPv4 or IPv6). + /// This is the group address visible to applications. + /// Validated at construction to be a multicast address. + pub overlay_group: OverlayMulticast, + + /// The underlay multicast group address (ff04::X). + /// Validated at construction to be within ff04::/64. + pub underlay_group: UnderlayMulticastIpv6, + + /// VNI for this multicast group (identifies the VPC/network context). + #[serde(default = "default_multicast_vni")] + pub vni: Vni, + + /// Metric for path selection (lower is better). + /// + /// Used for multi-rack replication optimization. + /// Excluded from identity (Hash/Eq) so that metric changes update an + /// existing entry rather than creating a duplicate. + #[serde(default)] + pub metric: u64, + + /// Optional source address for Source-Specific Multicast (S,G) routes. + /// `None` for Any-Source Multicast (*,G) routes. + #[serde(default)] + pub source: Option, +} + +impl MulticastOrigin { + /// Identity used for equality and hashing: the group, its underlay + /// mapping, VNI, and source. Excludes `metric`, a mutable path-selection + /// attribute, so a metric change updates an existing entry rather than + /// creating a duplicate. Routing both `PartialEq` and `Hash` through this + /// accessor keeps the field set defined once so the two cannot drift. + /// + /// This type is not used in ordered collections (BTreeSet). See #649 for + /// why adding `Ord` here would require more care. + fn identity( + &self, + ) -> ( + &OverlayMulticast, + &UnderlayMulticastIpv6, + &Vni, + &Option, + ) { + ( + &self.overlay_group, + &self.underlay_group, + &self.vni, + &self.source, + ) + } + + /// Return a stable string key for this origin's identity. + /// + /// Serializes only the identity fields, matching `PartialEq`/`Hash`, so a + /// keyed store overwrites the entry for an origin whose `metric` changed + /// rather than leaving a stale entry under the prior metric. Deriving the + /// key from [`MulticastOrigin::identity`] keeps it from drifting from + /// equality. + /// + /// # Errors + /// + /// Returns an error if the identity fields fail to serialize. + pub fn identity_key(&self) -> Result { + serde_json::to_string(&self.identity()) + } +} + +impl PartialEq for MulticastOrigin { + fn eq(&self, other: &Self) -> bool { + self.identity() == other.identity() + } +} + +impl std::hash::Hash for MulticastOrigin { + fn hash(&self, state: &mut H) { + self.identity().hash(state); + } +} + +// Locally-originated groups are operator-supplied and validated at the admin +// boundary. Before propagation on the wire the validated form is lowered to the +// plain `v4::MulticastOrigin`. The reverse promotion is deliberately absent: +// peer-supplied routes are stored in their plain form, +// trusted like tunnel and underlay routes. The one exception is the underlay +// group, which reaches DPD directly, so the exchange handler enforces its +// ff04::/64 invariant at import. +impl From<&MulticastOrigin> for v4::MulticastOrigin { + fn from(o: &MulticastOrigin) -> Self { + Self { + overlay_group: o.overlay_group.ip(), + underlay_group: o.underlay_group.ip(), + vni: o.vni.as_u32(), + metric: o.metric, + source: o.source, + } + } +} + +impl From for v4::MulticastOrigin { + fn from(o: MulticastOrigin) -> Self { + Self::from(&o) + } +} + +// Promote a peer-supplied wire origin to the validated form. Peer routes +// arrive as the plain `v4::MulticastOrigin`. Promotion enforces the invariants +// the rich type guarantees, the overlay group being multicast, the underlay +// group within ff04::/64, and the VNI within the 24-bit range, so an invalid +// origin is rejected at the exchange boundary before it can be stored or reach +// DPD. +impl TryFrom<&v4::MulticastOrigin> for MulticastOrigin { + type Error = MulticastOriginError; + + fn try_from(o: &v4::MulticastOrigin) -> Result { + Ok(Self { + overlay_group: OverlayMulticast::new(o.overlay_group)?, + underlay_group: UnderlayMulticastIpv6::new(o.underlay_group)?, + vni: Vni::new(o.vni)?, + metric: o.metric, + source: o.source, + }) + } +} + +impl TryFrom for MulticastOrigin { + type Error = MulticastOriginError; + + fn try_from(o: v4::MulticastOrigin) -> Result { + Self::try_from(&o) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::net::Ipv6Addr; + + #[test] + fn multicast_origin_rejects_bad_underlay() { + let json = serde_json::json!({ + "overlay_group": "233.252.0.1", + "underlay_group": "ff0e::1", + "vni": 77 + }); + let result: Result = serde_json::from_value(json); + assert!(result.is_err()); + } + + #[test] + fn multicast_origin_accepts_valid() { + let json = serde_json::json!({ + "overlay_group": "233.252.0.1", + "underlay_group": "ff04::1", + "vni": 77 + }); + let origin: MulticastOrigin = serde_json::from_value(json).unwrap(); + assert_eq!( + origin.underlay_group.ip(), + Ipv6Addr::new(0xff04, 0, 0, 0, 0, 0, 0, 1), + ); + } + + // Metric is excluded from the identity so a metric-only change updates an + // existing entry rather than duplicating it, mirroring the wire-type + // contract in `v4::MulticastOrigin`. + #[test] + fn multicast_origin_identity_excludes_metric() { + let base: MulticastOrigin = serde_json::from_value(serde_json::json!({ + "overlay_group": "233.252.0.1", + "underlay_group": "ff04::1", + "vni": 77, + "metric": 0 + })) + .unwrap(); + let mut other = base.clone(); + other.metric = 100; + assert_eq!(base, other); + } +} diff --git a/ddm-api-types/versions/src/peer_durations/mod.rs b/ddm-api-types/versions/src/peer_durations/mod.rs index 7a9776a99..a60e9bd95 100644 --- a/ddm-api-types/versions/src/peer_durations/mod.rs +++ b/ddm-api-types/versions/src/peer_durations/mod.rs @@ -2,7 +2,8 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -//! Version `PEER_DURATIONS` of the DDM Admin API. +//! Types from API version 2 (PEER_DURATIONS) that changed in version 3 +//! (MULTICAST_SUPPORT). //! //! Tracks how long each DDM peer has been in its current state and exposes //! that through the `/peers` endpoint with duration information. diff --git a/ddm-api/src/lib.rs b/ddm-api/src/lib.rs index 623b8376b..1a77a257a 100644 --- a/ddm-api/src/lib.rs +++ b/ddm-api/src/lib.rs @@ -4,6 +4,7 @@ use ddm_api_types_versions::latest; use ddm_api_types_versions::v1; +use ddm_api_types_versions::v2; use dropshot::HttpError; use dropshot::HttpResponseOk; use dropshot::HttpResponseUpdatedNoContent; @@ -26,6 +27,7 @@ api_versions!([ // | example for the next person. // v // (next_int, IDENT), + (3, MULTICAST_SUPPORT), (2, PEER_DURATIONS), (1, INITIAL), ]); @@ -49,23 +51,45 @@ pub trait DdmAdminApi { #[endpoint { method = GET, path = "/peers", - versions = VERSION_PEER_DURATIONS.., + versions = VERSION_MULTICAST_SUPPORT.. }] async fn get_peers( ctx: RequestContext, ) -> Result>, HttpError>; + /// Returns peers without interface name information. #[endpoint { method = GET, path = "/peers", - versions = ..VERSION_PEER_DURATIONS, + versions = VERSION_PEER_DURATIONS..VERSION_MULTICAST_SUPPORT + }] + async fn get_peers_v2( + ctx: RequestContext, + ) -> Result>, HttpError> { + let resp = Self::get_peers(ctx).await?; + let converted: HashMap = + resp.0.into_iter().map(|(k, v)| (k, v.into())).collect(); + Ok(HttpResponseOk(converted)) + } + + /// Returns peers without per-state duration or interface name information. + #[endpoint { + method = GET, + path = "/peers", + versions = ..VERSION_PEER_DURATIONS }] async fn get_peers_v1( ctx: RequestContext, ) -> Result>, HttpError> { let resp = Self::get_peers(ctx).await?; - let converted: HashMap = - resp.0.into_iter().map(|(k, v)| (k, v.into())).collect(); + let converted: HashMap = resp + .0 + .into_iter() + .map(|(k, v)| { + let v2_info: v2::db::PeerInfo = v.into(); + (k, v2_info.into()) + }) + .collect(); Ok(HttpResponseOk(converted)) } @@ -146,10 +170,45 @@ pub trait DdmAdminApi { request: TypedBody>, ) -> Result; + #[endpoint { + method = GET, + path = "/originated_multicast_groups", + versions = VERSION_MULTICAST_SUPPORT.. + }] + async fn get_originated_multicast_groups( + ctx: RequestContext, + ) -> Result>, HttpError>; + + #[endpoint { + method = GET, + path = "/multicast_groups", + versions = VERSION_MULTICAST_SUPPORT.. + }] + async fn get_multicast_groups( + ctx: RequestContext, + ) -> Result>, HttpError>; + #[endpoint { method = PUT, - path = "/sync", + path = "/multicast_group", + versions = VERSION_MULTICAST_SUPPORT.. }] + async fn advertise_multicast_groups( + ctx: RequestContext, + request: TypedBody>, + ) -> Result; + + #[endpoint { + method = DELETE, + path = "/multicast_group", + versions = VERSION_MULTICAST_SUPPORT.. + }] + async fn withdraw_multicast_groups( + ctx: RequestContext, + request: TypedBody>, + ) -> Result; + + #[endpoint { method = PUT, path = "/sync" }] async fn sync( ctx: RequestContext, ) -> Result; diff --git a/ddm-protocol/src/lib.rs b/ddm-protocol/src/lib.rs index 7585766ec..73fc3969b 100644 --- a/ddm-protocol/src/lib.rs +++ b/ddm-protocol/src/lib.rs @@ -8,6 +8,48 @@ use oxnet::{IpNet, Ipv4Net, Ipv6Net}; pub mod v2; pub mod v3; +pub mod v4; + +impl From for v4::Update { + fn from(value: v3::Update) -> Self { + Self { + underlay: value.underlay, + tunnel: value.tunnel, + // V3 has no multicast half. + multicast: None, + } + } +} + +impl From for v3::Update { + fn from(value: v4::Update) -> Self { + // The multicast half is not representable in the V3 wire form and is + // dropped here. A V3 peer would likewise ignore the unknown field. + Self { + underlay: value.underlay, + tunnel: value.tunnel, + } + } +} + +impl From for v4::PullResponse { + fn from(value: v3::PullResponse) -> Self { + Self { + underlay: value.underlay, + tunnel: value.tunnel, + multicast: None, + } + } +} + +impl From for v3::PullResponse { + fn from(value: v4::PullResponse) -> Self { + Self { + underlay: value.underlay, + tunnel: value.tunnel, + } + } +} impl From for v3::Update { fn from(value: v2::Update) -> Self { diff --git a/ddm-protocol/src/v3.rs b/ddm-protocol/src/v3.rs index 80cfcb925..688b76ff2 100644 --- a/ddm-protocol/src/v3.rs +++ b/ddm-protocol/src/v3.rs @@ -20,8 +20,8 @@ pub struct Update { impl Update { /// Build an `Update` whose underlay/tunnel halves carry the announcements - /// from `pr`. Used by [`pull`] to project a pull response back into the - /// update event stream. + /// from the [`PullResponse`] `pr`. Used by [`pull`] to project a pull + /// response back into the update event stream. pub fn announce(pr: PullResponse) -> Self { Self { underlay: pr.underlay.map(UnderlayUpdate::announce), diff --git a/ddm-protocol/src/v4.rs b/ddm-protocol/src/v4.rs new file mode 100644 index 000000000..a982ca369 --- /dev/null +++ b/ddm-protocol/src/v4.rs @@ -0,0 +1,423 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! ALL TYPES IN THIS FILE ARE FOR DDM PROTOCOL VERSION 4. THEY SHALL NEVER +//! CHANGE. THESE TYPES CAN BE REMOVED WHEN DDMV4 CLIENTS AND SERVERS NO LONGER +//! EXIST BUT THEIR DEFINITIONS SHALL NEVER CHANGE. +//! +//! Version 4 extends version 3 with multicast group subscription propagation +//! (RFD 488). The underlay and tunnel halves are unchanged from version 3 and +//! are reused directly. The multicast wire types are defined here as plain, +//! self-contained structures: this crate must stay free of `omicron-common`, +//! so the validated forms (`UnderlayMulticastIpv6`, `Vni`) used by the admin +//! and database layers are converted to and from these wire types at the +//! exchange boundary. + +use std::{ + collections::HashSet, + net::{IpAddr, Ipv6Addr}, +}; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::v3; + +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] +pub struct Update { + pub underlay: Option, + pub tunnel: Option, + pub multicast: Option, +} + +impl Update { + /// Build an `Update` whose halves carry the announcements from the + /// [`PullResponse`] `pr`. + pub fn announce(pr: PullResponse) -> Self { + Self { + underlay: pr.underlay.map(v3::UnderlayUpdate::announce), + tunnel: pr.tunnel.map(v3::TunnelUpdate::announce), + multicast: pr.multicast.map(MulticastUpdate::announce), + } + } +} + +impl From for Update { + fn from(u: v3::UnderlayUpdate) -> Self { + Update { + underlay: Some(u), + tunnel: None, + multicast: None, + } + } +} + +impl From for Update { + fn from(t: v3::TunnelUpdate) -> Self { + Update { + underlay: None, + tunnel: Some(t), + multicast: None, + } + } +} + +impl From for Update { + fn from(m: MulticastUpdate) -> Self { + Update { + underlay: None, + tunnel: None, + multicast: Some(m), + } + } +} + +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] +pub struct PullResponse { + pub underlay: Option>, + pub tunnel: Option>, + pub multicast: Option>, +} + +/// Multicast group subscription updates. +/// +/// Each entry carries a [`MulticastPathVector`] with the group origin and the +/// path vector used for loop detection. +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] +pub struct MulticastUpdate { + pub announce: HashSet, + pub withdraw: HashSet, +} + +impl MulticastUpdate { + pub fn announce(groups: HashSet) -> Self { + Self { + announce: groups, + ..Default::default() + } + } + pub fn withdraw(groups: HashSet) -> Self { + Self { + withdraw: groups, + ..Default::default() + } + } + + /// Add a hop to all path vectors in this update. + pub fn with_hop(&self, hop: MulticastPathHop) -> Self { + Self { + announce: self + .announce + .iter() + .map(|pv| pv.with_hop(hop.clone())) + .collect(), + withdraw: self + .withdraw + .iter() + .map(|pv| pv.with_hop(hop.clone())) + .collect(), + } + } +} + +/// Wire form of a multicast group origin. +/// +/// The validated counterpart (`ddm_api_types::net::MulticastOrigin`) carries an +/// `UnderlayMulticastIpv6` and a `Vni`. As a frozen wire type this form stays +/// unvalidated, a plain `Ipv6Addr` for the underlay group and a plain `u32` for +/// the VNI. Validation happens when converting into that counterpart at the +/// exchange boundary. +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] +pub struct MulticastOrigin { + /// The overlay multicast group address (IPv4 or IPv6). + pub overlay_group: IpAddr, + + /// The underlay multicast group address (ff04::X on the wire). + pub underlay_group: Ipv6Addr, + + /// VNI identifying the VPC/network context for this group. + #[serde(default)] + pub vni: u32, + + /// Metric for path selection (lower is better). Excluded from identity so + /// that metric changes update an existing entry rather than duplicating it. + #[serde(default)] + pub metric: u64, + + /// Optional source address for Source-Specific Multicast (S,G) routes. + /// `None` for Any-Source Multicast (*,G) routes. + #[serde(default)] + pub source: Option, +} + +impl MulticastOrigin { + /// Identity used for equality and hashing: the group, its underlay mapping, + /// VNI, and source. Excludes `metric`, a mutable path-selection attribute, + /// matching the validated `MulticastOrigin`. Routing both `PartialEq` and + /// `Hash` through this accessor keeps the field set defined once so the two + /// cannot drift. + fn identity(&self) -> (&IpAddr, &Ipv6Addr, &u32, &Option) { + ( + &self.overlay_group, + &self.underlay_group, + &self.vni, + &self.source, + ) + } +} + +impl PartialEq for MulticastOrigin { + fn eq(&self, other: &Self) -> bool { + self.identity() == other.identity() + } +} + +impl Eq for MulticastOrigin {} + +impl std::hash::Hash for MulticastOrigin { + fn hash(&self, state: &mut H) { + self.identity().hash(state); + } +} + +/// A single hop in the multicast path, carrying metadata for replication +/// optimization (RFD 488). +#[derive( + Debug, Clone, PartialEq, Eq, Hash, Deserialize, Serialize, JsonSchema, +)] +pub struct MulticastPathHop { + /// Router identifier (hostname). + pub router_id: String, + + /// The underlay address of this router (for replication targeting). + pub underlay_addr: Ipv6Addr, + + /// Number of downstream subscribers reachable via this hop. + #[serde(default)] + pub downstream_subscriber_count: u32, +} + +impl MulticastPathHop { + /// Create a hop with the given router identity and a zero subscriber count. + pub fn new(router_id: String, underlay_addr: Ipv6Addr) -> Self { + Self { + router_id, + underlay_addr, + downstream_subscriber_count: 0, + } + } +} + +/// Multicast group subscription announcement propagating through DDM. +#[derive( + Debug, Clone, PartialEq, Eq, Hash, Deserialize, Serialize, JsonSchema, +)] +pub struct MulticastPathVector { + /// The multicast group origin information. + pub origin: MulticastOrigin, + + /// The path from the original subscriber to the current router, ordered + /// from subscriber outward (subscriber router first). + pub path: Vec, +} + +impl MulticastPathVector { + /// Append a hop to this path vector. + pub fn with_hop(&self, hop: MulticastPathHop) -> Self { + let mut path = self.path.clone(); + path.push(hop); + Self { + origin: self.origin.clone(), + path, + } + } +} + +#[cfg(test)] +mod test { + use super::*; + + // Write out the JSON schema for the DDMv4 protocol to a file for + // validation. This should not change. + #[test] + fn test_ddm_v4_protocol() { + #[derive(JsonSchema)] + #[allow(dead_code)] + struct Protocol { + update: Update, + pull_response: PullResponse, + } + + let schema = schemars::schema_for!(Protocol); + expectorate::assert_contents( + "tests/output/ddm_v4_protocol.json", + &serde_json::to_string_pretty(&schema).unwrap(), + ); + } + + fn mcast_origin(overlay: &str, underlay: &str) -> MulticastOrigin { + MulticastOrigin { + overlay_group: overlay.parse().unwrap(), + underlay_group: underlay.parse().unwrap(), + vni: 77, + metric: 0, + source: None, + } + } + + fn multicast_update() -> MulticastUpdate { + let pv = MulticastPathVector { + origin: mcast_origin("233.252.0.1", "ff04::1"), + path: vec![MulticastPathHop::new( + "router-1".into(), + Ipv6Addr::LOCALHOST, + )], + }; + MulticastUpdate::announce([pv].into_iter().collect()) + } + + #[test] + fn v4_update_round_trips() { + let update = Update { + underlay: None, + tunnel: None, + multicast: Some(multicast_update()), + }; + let json = serde_json::to_string(&update).unwrap(); + let back: Update = serde_json::from_str(&json).unwrap(); + assert!(back.multicast.is_some()); + assert_eq!(back.multicast.unwrap().announce.len(), 1); + } + + #[test] + fn v4_update_deserializes_as_v3_drops_multicast() { + let update = Update { + underlay: None, + tunnel: None, + multicast: Some(multicast_update()), + }; + let json = serde_json::to_string(&update).unwrap(); + // A v3 peer deserializes this as a v3 update, silently dropping the + // unknown multicast field. + let v3: v3::Update = serde_json::from_str(&json).unwrap(); + assert!(v3.underlay.is_none()); + assert!(v3.tunnel.is_none()); + } + + // A v4 node reading a populated v3 update keeps the underlay and tunnel + // halves and defaults the absent multicast half to None. + #[test] + fn populated_v3_update_deserializes_as_v4() { + let v3 = v3::Update { + underlay: Some(underlay_update()), + tunnel: Some(tunnel_update()), + }; + let json = serde_json::to_string(&v3).unwrap(); + let update: Update = serde_json::from_str(&json).unwrap(); + assert!(update.underlay.is_some()); + assert!(update.tunnel.is_some()); + assert!(update.multicast.is_none()); + } + + #[test] + fn v4_pull_response_round_trips() { + let pv = MulticastPathVector { + origin: mcast_origin("ff0e::1", "ff04::2"), + path: vec![], + }; + let resp = PullResponse { + underlay: None, + tunnel: None, + multicast: Some([pv].into_iter().collect()), + }; + let json = serde_json::to_string(&resp).unwrap(); + let back: PullResponse = serde_json::from_str(&json).unwrap(); + assert!(back.multicast.is_some()); + } + + #[test] + fn v4_pull_response_deserializes_as_v3() { + let pv = MulticastPathVector { + origin: mcast_origin("233.252.0.1", "ff04::1"), + path: vec![], + }; + let resp = PullResponse { + underlay: None, + tunnel: None, + multicast: Some([pv].into_iter().collect()), + }; + let json = serde_json::to_string(&resp).unwrap(); + // A v3 peer drops the multicast field. + let v3: v3::PullResponse = serde_json::from_str(&json).unwrap(); + assert!(v3.underlay.is_none()); + assert!(v3.tunnel.is_none()); + } + + #[test] + fn from_conversions_strip_multicast() { + let update = Update { + underlay: None, + tunnel: None, + multicast: Some(multicast_update()), + }; + // Downconvert to v3 for an older peer, then back. Multicast has no v3 + // representation, so the round trip drops it. + let v3 = v3::Update::from(update); + let back = Update::from(v3); + assert!(back.multicast.is_none()); + } + + fn underlay_update() -> v3::UnderlayUpdate { + let pv = v3::PathVector { + destination: "fd00::/64".parse().unwrap(), + path: vec!["router-1".into()], + }; + v3::UnderlayUpdate::announce([pv].into_iter().collect()) + } + + fn tunnel_update() -> v3::TunnelUpdate { + let origin = v3::TunnelOrigin { + overlay_prefix: "10.0.0.0/24".parse().unwrap(), + boundary_addr: Ipv6Addr::LOCALHOST, + vni: 77, + metric: 0, + }; + v3::TunnelUpdate::announce([origin].into_iter().collect()) + } + + // A v4 update carrying all three halves must keep its underlay and tunnel + // halves intact when downconverted for older peers, while the multicast + // half (which has no v3 or v2 wire form) is dropped. + #[test] + fn mixed_update_down_conversion_preserves_underlay_and_tunnel() { + let update = Update { + underlay: Some(underlay_update()), + tunnel: Some(tunnel_update()), + multicast: Some(multicast_update()), + }; + + let v3 = v3::Update::from(update.clone()); + assert!(v3.underlay.is_some()); + assert!(v3.tunnel.is_some()); + assert_eq!(v3.underlay.as_ref().unwrap().announce.len(), 1); + assert_eq!(v3.tunnel.as_ref().unwrap().announce.len(), 1); + + let v2 = crate::v2::Update::from(v3); + assert!(v2.underlay.is_some()); + assert!(v2.tunnel.is_some()); + assert_eq!(v2.underlay.unwrap().announce.len(), 1); + assert_eq!(v2.tunnel.unwrap().announce.len(), 1); + } + + // Metric is excluded from the wire identity so a metric-only change updates + // an existing entry rather than duplicating it inside a HashSet. + #[test] + fn multicast_origin_identity_excludes_metric() { + let mut a = mcast_origin("233.252.0.1", "ff04::1"); + let mut b = a.clone(); + a.metric = 0; + b.metric = 100; + assert_eq!(a, b); + } +} diff --git a/ddm-protocol/tests/output/ddm_v4_protocol.json b/ddm-protocol/tests/output/ddm_v4_protocol.json new file mode 100644 index 000000000..0b10c49bf --- /dev/null +++ b/ddm-protocol/tests/output/ddm_v4_protocol.json @@ -0,0 +1,352 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Protocol", + "type": "object", + "required": [ + "pull_response", + "update" + ], + "properties": { + "pull_response": { + "$ref": "#/definitions/PullResponse" + }, + "update": { + "$ref": "#/definitions/Update" + } + }, + "definitions": { + "IpNet": { + "oneOf": [ + { + "title": "v4", + "allOf": [ + { + "$ref": "#/definitions/Ipv4Net" + } + ] + }, + { + "title": "v6", + "allOf": [ + { + "$ref": "#/definitions/Ipv6Net" + } + ] + } + ], + "x-rust-type": { + "crate": "oxnet", + "path": "oxnet::IpNet", + "version": "0.1.0" + } + }, + "Ipv4Net": { + "title": "An IPv4 subnet", + "description": "An IPv4 subnet, including prefix and prefix length", + "examples": [ + "192.168.1.0/24" + ], + "type": "string", + "pattern": "^(([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])/([0-9]|1[0-9]|2[0-9]|3[0-2])$", + "x-rust-type": { + "crate": "oxnet", + "path": "oxnet::Ipv4Net", + "version": "0.1.0" + } + }, + "Ipv6Net": { + "title": "An IPv6 subnet", + "description": "An IPv6 subnet, including prefix and subnet mask", + "examples": [ + "fd12:3456::/64" + ], + "type": "string", + "pattern": "^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))\\/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8])$", + "x-rust-type": { + "crate": "oxnet", + "path": "oxnet::Ipv6Net", + "version": "0.1.0" + } + }, + "MulticastOrigin": { + "description": "Wire form of a multicast group origin.\n\nThe validated counterpart (`ddm_api_types::net::MulticastOrigin`) carries an `UnderlayMulticastIpv6` and a `Vni`. As a frozen wire type this form stays unvalidated, a plain `Ipv6Addr` for the underlay group and a plain `u32` for the VNI. Validation happens when converting into that counterpart at the exchange boundary.", + "type": "object", + "required": [ + "overlay_group", + "underlay_group" + ], + "properties": { + "metric": { + "description": "Metric for path selection (lower is better). Excluded from identity so that metric changes update an existing entry rather than duplicating it.", + "default": 0, + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "overlay_group": { + "description": "The overlay multicast group address (IPv4 or IPv6).", + "type": "string", + "format": "ip" + }, + "source": { + "description": "Optional source address for Source-Specific Multicast (S,G) routes. `None` for Any-Source Multicast (*,G) routes.", + "default": null, + "type": [ + "string", + "null" + ], + "format": "ip" + }, + "underlay_group": { + "description": "The underlay multicast group address (ff04::X on the wire).", + "type": "string", + "format": "ipv6" + }, + "vni": { + "description": "VNI identifying the VPC/network context for this group.", + "default": 0, + "type": "integer", + "format": "uint32", + "minimum": 0.0 + } + } + }, + "MulticastPathHop": { + "description": "A single hop in the multicast path, carrying metadata for replication optimization (RFD 488).", + "type": "object", + "required": [ + "router_id", + "underlay_addr" + ], + "properties": { + "downstream_subscriber_count": { + "description": "Number of downstream subscribers reachable via this hop.", + "default": 0, + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "router_id": { + "description": "Router identifier (hostname).", + "type": "string" + }, + "underlay_addr": { + "description": "The underlay address of this router (for replication targeting).", + "type": "string", + "format": "ipv6" + } + } + }, + "MulticastPathVector": { + "description": "Multicast group subscription announcement propagating through DDM.", + "type": "object", + "required": [ + "origin", + "path" + ], + "properties": { + "origin": { + "description": "The multicast group origin information.", + "allOf": [ + { + "$ref": "#/definitions/MulticastOrigin" + } + ] + }, + "path": { + "description": "The path from the original subscriber to the current router, ordered from subscriber outward (subscriber router first).", + "type": "array", + "items": { + "$ref": "#/definitions/MulticastPathHop" + } + } + } + }, + "MulticastUpdate": { + "description": "Multicast group subscription updates.\n\nEach entry carries a [`MulticastPathVector`] with the group origin and the path vector used for loop detection.", + "type": "object", + "required": [ + "announce", + "withdraw" + ], + "properties": { + "announce": { + "type": "array", + "items": { + "$ref": "#/definitions/MulticastPathVector" + }, + "uniqueItems": true + }, + "withdraw": { + "type": "array", + "items": { + "$ref": "#/definitions/MulticastPathVector" + }, + "uniqueItems": true + } + } + }, + "PathVector": { + "type": "object", + "required": [ + "destination", + "path" + ], + "properties": { + "destination": { + "$ref": "#/definitions/Ipv6Net" + }, + "path": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "PullResponse": { + "type": "object", + "properties": { + "multicast": { + "type": [ + "array", + "null" + ], + "items": { + "$ref": "#/definitions/MulticastPathVector" + }, + "uniqueItems": true + }, + "tunnel": { + "type": [ + "array", + "null" + ], + "items": { + "$ref": "#/definitions/TunnelOrigin" + }, + "uniqueItems": true + }, + "underlay": { + "type": [ + "array", + "null" + ], + "items": { + "$ref": "#/definitions/PathVector" + }, + "uniqueItems": true + } + } + }, + "TunnelOrigin": { + "type": "object", + "required": [ + "boundary_addr", + "overlay_prefix", + "vni" + ], + "properties": { + "boundary_addr": { + "type": "string", + "format": "ipv6" + }, + "metric": { + "default": 0, + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "overlay_prefix": { + "$ref": "#/definitions/IpNet" + }, + "vni": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + } + } + }, + "TunnelUpdate": { + "type": "object", + "required": [ + "announce", + "withdraw" + ], + "properties": { + "announce": { + "type": "array", + "items": { + "$ref": "#/definitions/TunnelOrigin" + }, + "uniqueItems": true + }, + "withdraw": { + "type": "array", + "items": { + "$ref": "#/definitions/TunnelOrigin" + }, + "uniqueItems": true + } + } + }, + "UnderlayUpdate": { + "type": "object", + "required": [ + "announce", + "withdraw" + ], + "properties": { + "announce": { + "type": "array", + "items": { + "$ref": "#/definitions/PathVector" + }, + "uniqueItems": true + }, + "withdraw": { + "type": "array", + "items": { + "$ref": "#/definitions/PathVector" + }, + "uniqueItems": true + } + } + }, + "Update": { + "type": "object", + "properties": { + "multicast": { + "anyOf": [ + { + "$ref": "#/definitions/MulticastUpdate" + }, + { + "type": "null" + } + ] + }, + "tunnel": { + "anyOf": [ + { + "$ref": "#/definitions/TunnelUpdate" + }, + { + "type": "null" + } + ] + }, + "underlay": { + "anyOf": [ + { + "$ref": "#/definitions/UnderlayUpdate" + }, + { + "type": "null" + } + ] + } + } + } + } +} \ No newline at end of file diff --git a/ddm/Cargo.toml b/ddm/Cargo.toml index 1168489a9..919a093a5 100644 --- a/ddm/Cargo.toml +++ b/ddm/Cargo.toml @@ -5,6 +5,7 @@ edition = "2024" [dev-dependencies] pretty_assertions.workspace = true +tempfile = "3" expectorate.workspace = true [dependencies] @@ -18,6 +19,7 @@ thiserror.workspace = true dropshot.workspace = true schemars.workspace = true tokio.workspace = true +futures.workspace = true anyhow.workspace = true camino.workspace = true hyper.workspace = true @@ -28,6 +30,7 @@ sled.workspace = true mg-common.workspace = true port-file.workspace = true ddm-api-types.workspace = true +ddm-api-types-versions.workspace = true ddm-protocol.workspace = true chrono.workspace = true omicron-common.workspace = true @@ -44,7 +47,8 @@ libnet = { workspace = true, optional = true } dpd-client = { workspace = true, optional = true } opte-ioctl = { workspace = true, optional = true } oxide-vpc = { workspace = true, optional = true } +reqwest = { workspace = true, optional = true } [features] default = ["backend"] -backend = ["dep:libnet", "dep:dpd-client", "dep:opte-ioctl", "dep:oxide-vpc"] +backend = ["dep:libnet", "dep:dpd-client", "dep:opte-ioctl", "dep:oxide-vpc", "dep:reqwest"] diff --git a/ddm/src/admin.rs b/ddm/src/admin.rs index 21d373f4d..27610e2cc 100644 --- a/ddm/src/admin.rs +++ b/ddm/src/admin.rs @@ -8,9 +8,9 @@ use camino::Utf8PathBuf; use ddm_api::DdmAdminApi; use ddm_api::ddm_admin_api_mod; use ddm_api_types::admin::{EnableStatsRequest, ExpirePathParams, PrefixMap}; -use ddm_api_types::db::{PeerInfo, TunnelRoute}; +use ddm_api_types::db::{MulticastRoute, PeerInfo, TunnelRoute}; use ddm_api_types::exchange::PathVector; -use ddm_api_types::net::TunnelOrigin; +use ddm_api_types::net::{MulticastOrigin, TunnelOrigin}; use dropshot::ApiDescription; use dropshot::ApiDescriptionBuildErrors; use dropshot::ConfigDropshot; @@ -50,6 +50,11 @@ pub struct HandlerContext { pub event_channels: Vec>, pub db: Db, pub stats: Arc, + /// Per-interface state machine contexts shared with the multicast sweep, + /// seeded from the running state machines and read by the `/peers` view. + /// + /// Under the `--api-only` flag there are no state machines, so the set is + /// empty. pub peers: Vec, pub stats_handler: Arc>>>, pub log: Logger, @@ -129,27 +134,7 @@ impl DdmAdminApi for DdmAdminApiImpl { async fn get_peers( ctx: RequestContext, ) -> Result>, HttpError> { - let ctx = lock!(ctx.context()); - let mut result = HashMap::new(); - for sm in &ctx.peers { - // Compute status first so peer_status() never runs while we hold - // any of the InterfaceState mutexes below. - let status = sm.iface.peer_status(); - let if_index = *lock!(sm.iface.if_index); - let Some(peer) = lock!(sm.iface.peer_identity).clone() else { - continue; - }; - result.insert( - if_index, - PeerInfo { - status, - addr: peer.addr, - host: peer.hostname, - kind: peer.kind, - }, - ); - } - Ok(HttpResponseOk(result)) + Ok(HttpResponseOk(do_get_peers(ctx.context()))) } async fn expire_peer( @@ -369,6 +354,76 @@ impl DdmAdminApi for DdmAdminApiImpl { Ok(HttpResponseUpdatedNoContent()) } + async fn get_originated_multicast_groups( + ctx: RequestContext, + ) -> Result>, HttpError> { + let ctx = lock!(ctx.context()); + let originated = ctx + .db + .originated_mcast() + .map_err(|e| HttpError::for_internal_error(e.to_string()))?; + Ok(HttpResponseOk(originated)) + } + + async fn get_multicast_groups( + ctx: RequestContext, + ) -> Result>, HttpError> { + let ctx = lock!(ctx.context()); + let imported = ctx.db.imported_mcast(); + Ok(HttpResponseOk(imported)) + } + + async fn advertise_multicast_groups( + ctx: RequestContext, + request: TypedBody>, + ) -> Result { + let ctx = lock!(ctx.context()); + let groups = request.into_inner(); + slog::info!(ctx.log, "advertise multicast groups: {groups:#?}"); + ctx.db + .originate_mcast(&groups) + .map_err(|e| HttpError::for_internal_error(e.to_string()))?; + + for e in &ctx.event_channels { + e.send(Event::Admin(AdminEvent::AnnounceMulticast(groups.clone()))) + .map_err(|e| { + HttpError::for_internal_error(format!( + "admin event send: {e}" + )) + })?; + } + + Ok(HttpResponseUpdatedNoContent()) + } + + async fn withdraw_multicast_groups( + ctx: RequestContext, + request: TypedBody>, + ) -> Result { + let ctx = lock!(ctx.context()); + let groups = request.into_inner(); + slog::info!(ctx.log, "withdraw multicast groups: {groups:#?}"); + // The modification is applied before any event is enqueued, and each + // state machine revalidates reachability when it processes the + // event. An import racing this request cannot be withdrawn against + // stale state, since the revalidation reads post-modification database + // state. The modification is idempotent, so a client retry is safe. + ctx.db + .withdraw_mcast(&groups) + .map_err(|e| HttpError::for_internal_error(e.to_string()))?; + + for e in &ctx.event_channels { + e.send(Event::Admin(AdminEvent::WithdrawMulticast(groups.clone()))) + .map_err(|e| { + HttpError::for_internal_error(format!( + "admin event send: {e}" + )) + })?; + } + + Ok(HttpResponseUpdatedNoContent()) + } + async fn sync( ctx: RequestContext, ) -> Result { @@ -436,3 +491,35 @@ pub fn api_description() { ddm_admin_api_mod::api_description::() } + +/// Snapshot the current peers, keyed by interface index. +/// +/// Reads the per-interface state machine contexts. +/// +/// Under the `--api-only` flag there are no state machines, so the map is empty. +pub(crate) fn do_get_peers( + ctx: &Arc>, +) -> HashMap { + let ctx = lock!(ctx); + ctx.peers + .iter() + .filter_map(|sm| { + // Compute status first so peer_status() never runs while we hold + // any of the InterfaceState mutexes below. + let status = sm.iface.peer_status(); + let peer = lock!(sm.iface.peer_identity).clone()?; + let if_index = *lock!(sm.iface.if_index); + let if_name = lock!(sm.iface.if_name).clone(); + Some(( + if_index, + PeerInfo { + status, + addr: peer.addr, + host: peer.hostname, + kind: peer.kind, + if_name: (!if_name.is_empty()).then_some(if_name), + }, + )) + }) + .collect() +} diff --git a/ddm/src/db.rs b/ddm/src/db.rs index d72fd7b20..ee568b1a5 100644 --- a/ddm/src/db.rs +++ b/ddm/src/db.rs @@ -2,8 +2,8 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use ddm_api_types::db::TunnelRoute; -use ddm_api_types::net::TunnelOrigin; +use ddm_api_types::db::{MulticastRoute, TunnelRoute}; +use ddm_api_types::net::{MulticastOrigin, TunnelOrigin}; use mg_common::lock; use oxnet::{IpNet, Ipv6Net}; use schemars::JsonSchema; @@ -21,6 +21,10 @@ const ORIGINATE: &str = "originate"; /// tunnel endpoints. const TUNNEL_ORIGINATE: &str = "tunnel_originate"; +/// The handle used to open a persistent key-value tree for originated +/// multicast groups. +const MCAST_ORIGINATE: &str = "mcast_originate"; + #[derive(thiserror::Error, Debug)] pub enum Error { #[error("datastore error {0}")] @@ -43,10 +47,27 @@ pub struct Db { log: Logger, } +/// The realized change to the imported multicast set after applying an update. +/// +/// This holds the routes that actually became present or absent, computed as +/// the set difference between the pre- and post-update state rather than the +/// raw request. Downstream consumers reconcile only these groups, so an update +/// that re-imports an existing route or withdraws an absent one yields an empty +/// delta and triggers no work. +#[derive(Debug, Default, Clone)] +pub struct McastRibDelta { + /// Routes newly present after the update. + pub added: HashSet, + + /// Routes no longer present after the update. + pub removed: HashSet, +} + #[derive(Default, Clone)] pub struct DbData { pub imported: HashSet, pub imported_tunnel: HashSet, + pub imported_mcast: HashSet, } const _: () = { @@ -62,6 +83,7 @@ impl Db { log, }) } + pub fn dump(&self) -> DbData { lock!(self.data).clone() } @@ -82,6 +104,30 @@ impl Db { lock!(self.data).imported_tunnel.len() } + pub fn imported_mcast(&self) -> HashSet { + lock!(self.data).imported_mcast.clone() + } + + pub fn imported_mcast_count(&self) -> usize { + lock!(self.data).imported_mcast.len() + } + + /// Underlay groups imported via `nexthop`, deduplicated. + pub fn mcast_groups_for_nexthop( + &self, + nexthop: Ipv6Addr, + ) -> HashSet { + // Filter under the lock so the caller never clones the full imported + // set just to keep one peer's routes. Non-destructive analog of the + // next-hop filter in `remove_nexthop_routes`. + lock!(self.data) + .imported_mcast + .iter() + .filter(|route| route.nexthop == nexthop) + .map(|route| route.origin.underlay_group.ip()) + .collect() + } + pub fn import(&self, r: &HashSet) { lock!(self.data).imported.extend(r.clone()); } @@ -104,6 +150,73 @@ impl Db { } } + /// Atomically import and delete multicast routes under a single lock, + /// returning the effective [`McastRibDelta`] against the state before + /// any modification. + /// + /// The single lock avoids a TOCTOU race where concurrent modifications + /// between separate lock acquisitions could produce an incorrect delta. + /// Callers that redistribute the update also need post-modification + /// reachability and use + /// [`Db::update_imported_mcast_with_reachability`] instead. + pub fn update_imported_mcast( + &self, + import: &HashSet, + remove: &HashSet, + ) -> McastRibDelta { + Self::apply_imported_mcast(&mut lock!(self.data), import, remove) + } + + /// [`Db::update_imported_mcast`] variant that also captures a + /// [`MulticastReachability`] snapshot of post-modification reachability. + /// + /// The imported set is captured under the same lock as the modification, + /// so downstream withdrawal reconciliation cannot observe imported state + /// older than the modification that produced it. Callers that do not + /// redistribute have no reconciliation to feed and skip this variant's + /// imported-set clone and persistent origin read. + pub fn update_imported_mcast_with_reachability( + &self, + import: &HashSet, + remove: &HashSet, + ) -> (McastRibDelta, MulticastReachability) { + let (delta, imported) = { + let mut data = lock!(self.data); + let delta = Self::apply_imported_mcast(&mut data, import, remove); + (delta, data.imported_mcast.clone()) + }; + + // Persistent origins are not touched by this method, so reading them + // outside the lock still yields a post-modification snapshot. The + // added tree scan is acceptable, sled caches the tree and it stays + // small. + (delta, self.reachability_snapshot(imported)) + } + + /// Apply `import` and `remove` to the imported multicast set under the + /// caller-held lock, returning the effective delta. + fn apply_imported_mcast( + data: &mut DbData, + import: &HashSet, + remove: &HashSet, + ) -> McastRibDelta { + let before = data.imported_mcast.clone(); + // Route identity excludes the path, so `insert` would keep a stale + // path on re-import. `replace` lets the newest path win. + for x in import { + data.imported_mcast.replace(x.clone()); + } + + for x in remove { + data.imported_mcast.remove(x); + } + + let added = data.imported_mcast.difference(&before).cloned().collect(); + let removed = + before.difference(&data.imported_mcast).cloned().collect(); + McastRibDelta { added, removed } + } + pub fn originate(&self, prefixes: &HashSet) -> Result<(), Error> { let tree = self.persistent_data.open_tree(ORIGINATE)?; for p in prefixes { @@ -126,76 +239,117 @@ impl Db { Ok(()) } - pub fn originated(&self) -> Result, Error> { - let tree = self.persistent_data.open_tree(ORIGINATE)?; + /// Persist multicast origins for advertisement to peers. + pub fn originate_mcast( + &self, + origins: &HashSet, + ) -> Result<(), Error> { + let tree = self.persistent_data.open_tree(MCAST_ORIGINATE)?; + for o in origins { + // Key by the metric-excluded identity, storing the full origin as + // the value. `MulticastOrigin` equality ignores `metric`, so keying + // by identity lets a re-origination with a changed metric overwrite + // the stored entry instead of leaving a stale one under the old + // metric. + tree.insert( + o.identity_key()?.as_str(), + serde_json::to_string(o)?.as_str(), + )?; + } + tree.flush()?; + Ok(()) + } + + /// Scan a persistent origin tree with `parse`, skipping entries that + /// fail to read or parse. `kind` names the entry kind for log context. + fn scan_origin_tree( + &self, + tree: &str, + kind: &str, + parse: impl Fn(&[u8], &[u8]) -> Result, + ) -> Result, Error> + where + T: Eq + std::hash::Hash, + { + let tree = self.persistent_data.open_tree(tree)?; let result = tree - .scan_prefix(vec![]) + .iter() .filter_map(|item| { - let (key, _value) = match item { + let (key, value) = match item { Ok(item) => item, Err(e) => { error!( self.log, - "db: error ddm originated prefix: {e}" + "db: error fetching ddm {kind} entry: {e}" ); return None; } }; - Some(match Ipv6Net::from_db_key(&key) { - Ok(item) => item, + match parse(key.as_ref(), value.as_ref()) { + Ok(item) => Some(item), Err(e) => { - error!( - self.log, - "db: error parsing ddm origin entry value: {e}" - ); - return None; + error!(self.log, "db: error parsing ddm {kind}: {e}"); + None } - }) + } }) .collect(); Ok(result) } + pub fn originated(&self) -> Result, Error> { + self.scan_origin_tree(ORIGINATE, "origin prefix", |key, _value| { + Ipv6Net::from_db_key(key).map_err(|e| Error::DbKey(e.to_string())) + }) + } + pub fn originated_count(&self) -> Result { Ok(self.originated()?.len()) } pub fn originated_tunnel(&self) -> Result, Error> { - let tree = self.persistent_data.open_tree(TUNNEL_ORIGINATE)?; - let result = tree - .scan_prefix(vec![]) - .filter_map(|item| { - let (key, _value) = match item { - Ok(item) => item, - Err(e) => { - error!( - self.log, - "db: error fetching ddm tunnel origin entry: {e}" - ); - return None; - } - }; - let value = String::from_utf8_lossy(&key); - let value: TunnelOrigin = match serde_json::from_str(&value) { - Ok(item) => item, - Err(e) => { - error!( - self.log, - "db: error parsing ddm tunnel origin: {e}" - ); - return None; - } - }; - Some(value) - }) - .collect(); - Ok(result) + self.scan_origin_tree(TUNNEL_ORIGINATE, "tunnel origin", |key, _v| { + Ok(serde_json::from_slice(key)?) + }) } pub fn originated_tunnel_count(&self) -> Result { Ok(self.originated_tunnel()?.len()) } + /// Multicast origins originated locally. + /// + /// Each origin is keyed by its metric-excluded identity and stored as the + /// value, so the current metric is read back from the value rather than the + /// key. + /// + /// This iterates the tree directly rather than going through + /// [`Db::scan_origin_tree`], which skips entries that fail to read or + /// parse. Withdrawal reconciliation treats the result as the complete + /// origin set, so a silently skipped entry could become a false final + /// withdrawal. Here any per-entry failure fails the whole read. The + /// caller surfaces that by degrading the snapshot's origin set, and + /// reconciliation then drops the withdrawal rather than treating the + /// missing origins as truly gone. + /// + /// # Errors + /// + /// Returns an error if the tree cannot be opened or any entry fails to + /// read or parse. + pub fn originated_mcast(&self) -> Result, Error> { + let tree = self.persistent_data.open_tree(MCAST_ORIGINATE)?; + tree.iter() + .map(|item| { + let (_key, value) = item?; + Ok(serde_json::from_slice(&value)?) + }) + .collect() + } + + pub fn originated_mcast_count(&self) -> Result { + Ok(self.originated_mcast()?.len()) + } + pub fn withdraw(&self, prefixes: &HashSet) -> Result<(), Error> { let tree = self.persistent_data.open_tree(ORIGINATE)?; for p in prefixes { @@ -218,33 +372,116 @@ impl Db { Ok(()) } + /// Remove persisted multicast origins. + /// + /// State machines revalidate reachability at processing time via + /// [`Db::multicast_reachability`] rather than from a snapshot captured + /// here. + /// + /// The modification lands before any event is enqueued, so the + /// processing-time snapshot is guaranteed to reflect this removal. + pub fn withdraw_mcast( + &self, + origins: &HashSet, + ) -> Result<(), Error> { + let tree = self.persistent_data.open_tree(MCAST_ORIGINATE)?; + for o in origins { + // Remove by identity so a withdraw matches regardless of the metric + // the origin was advertised with (e.g. the CLI's default metric). + tree.remove(o.identity_key()?.as_str())?; + } + tree.flush()?; + Ok(()) + } + + /// Capture current multicast reachability for processing-time + /// revalidation. An event is enqueued only after its modification completes, + /// so a snapshot taken while processing that event is post-modification. It + /// also observes any later modification, which lets a state machine avoid + /// acting on reachability that has since been restored. + pub fn multicast_reachability(&self) -> MulticastReachability { + // Same lock-ordering discipline as `update_imported_mcast`: clone + // `imported_mcast` under the data lock and read persistent origins + // after dropping it, since the two sources are not co-modified. + let imported = lock!(self.data).imported_mcast.clone(); + self.reachability_snapshot(imported) + } + pub fn remove_nexthop_routes( &self, nexthop: Ipv6Addr, - ) -> (HashSet, HashSet) { - let mut data = lock!(self.data); - // Routes are generally held in sets to prevent duplication and provide - // handy set-algebra operations. - let mut removed = HashSet::new(); - for x in &data.imported { - if x.nexthop == nexthop { - removed.insert(x.clone()); + ) -> RemovedNexthopRoutes { + let (removed, tnl_removed, mcast_removed, imported_mcast) = { + let mut data = lock!(self.data); + let mut removed = HashSet::new(); + for x in &data.imported { + if x.nexthop == nexthop { + removed.insert(x.clone()); + } + } + for x in &removed { + data.imported.remove(x); + } + + let mut tnl_removed = HashSet::new(); + for x in &data.imported_tunnel { + if x.nexthop == nexthop { + tnl_removed.insert(*x); + } + } + for x in &tnl_removed { + data.imported_tunnel.remove(x); } - } - for x in &removed { - data.imported.remove(x); - } - let mut tnl_removed = HashSet::new(); - for x in &data.imported_tunnel { - if x.nexthop == nexthop { - tnl_removed.insert(*x); + let mut mcast_removed = HashSet::new(); + for x in &data.imported_mcast { + if x.nexthop == nexthop { + mcast_removed.insert(x.clone()); + } + } + for x in &mcast_removed { + data.imported_mcast.remove(x); } + + let imported_mcast = data.imported_mcast.clone(); + (removed, tnl_removed, mcast_removed, imported_mcast) + }; + + // Persistent origins are not touched by this method, so reading them + // outside the lock still yields a post-modification snapshot. + RemovedNexthopRoutes { + underlay: removed, + tunnel: tnl_removed, + multicast: mcast_removed, + mcast_reachability: self.reachability_snapshot(imported_mcast), } - for x in &tnl_removed { - data.imported_tunnel.remove(x); + } + + /// Build a post-modification reachability snapshot around an imported + /// set captured under the data lock, taking ownership so no further + /// clone is needed. A persistent origin read failure degrades + /// `originated` to the empty set and marks the snapshot, so consumers + /// know the empty origin set is a read failure rather than a real + /// absence. + fn reachability_snapshot( + &self, + imported: HashSet, + ) -> MulticastReachability { + match self.originated_mcast() { + Ok(originated) => MulticastReachability { + imported, + originated, + origins_degraded: false, + }, + Err(e) => { + error!(self.log, "read remaining multicast origins: {e}"); + MulticastReachability { + imported, + originated: HashSet::new(), + origins_degraded: true, + } + } } - (removed, tnl_removed) } pub fn routes_by_vector( @@ -263,6 +500,48 @@ impl Db { } } +/// Routes withdrawn for a next hop, grouped by route family. +pub struct RemovedNexthopRoutes { + pub underlay: HashSet, + pub tunnel: HashSet, + pub multicast: HashSet, + /// Post-modification multicast reachability snapshot captured under the same + /// removal that produced `multicast`. Downstream reconciliation reads + /// viable paths from here rather than reading the database again. + pub mcast_reachability: MulticastReachability, +} + +/// A snapshot of multicast reachability: imported routes and local origins. +/// +/// Only `Db` methods construct this. A snapshot is either captured by the +/// modification that produced a set of withdrawals or read at processing time +/// via [`Db::multicast_reachability`]. In both cases, it describes state no +/// older than that modification, since the event that triggers a +/// processing-time read is enqueued only after the modification completes. +#[derive(Debug, Clone)] +pub struct MulticastReachability { + imported: HashSet, + originated: HashSet, + origins_degraded: bool, +} + +impl MulticastReachability { + pub fn imported(&self) -> &HashSet { + &self.imported + } + + pub fn originated(&self) -> &HashSet { + &self.originated + } + + /// Whether the persistent origin read failed, degrading `originated` to + /// the empty set. A degraded snapshot cannot distinguish a withdrawn + /// origin from an unread one. + pub fn origins_degraded(&self) -> bool { + self.origins_degraded + } +} + #[derive( Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema, )] diff --git a/ddm/src/discovery/mod.rs b/ddm/src/discovery/mod.rs index 3f0167e59..bdccb0087 100644 --- a/ddm/src/discovery/mod.rs +++ b/ddm/src/discovery/mod.rs @@ -14,10 +14,9 @@ //! //! [`Version`] and [`DiscoveryError`] are platform-agnostic and stay in this //! module so the state machine type definitions in [`crate::sm`] continue to -//! compile when the routing runtime is gated out (e.g. Linux test fixtures -//! running `ddmd` with `--api-only`). The runtime helpers that drive -//! the protocol over UDPv6 sockets live in the [`runtime`] submodule and -//! are illumos-only. +//! compile when the routing runtime is gated out (e.g. a non-illumos `ddmd` +//! running with `--api-only`). The runtime helpers that drive the protocol +//! over UDPv6 sockets live in the `runtime` submodule and are illumos-only. //! //! ## Protocol //! @@ -80,7 +79,7 @@ //! 1 2 3 //! 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 //! +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -//! | version |S A r r r r r r| router kind | hostname len | +//! | version |S A C r r r r r| router kind | hostname len | //! +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ //! | hostname : //! +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ @@ -88,13 +87,15 @@ //! +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ //! ``` //! -//! The first byte indicates the version. The only valid version at present is -//! version 1. The second byte is a flags bitfield. The first position `S` -//! indicates a solicitation. The second position `A` indicates and -//! advertisement. All other positions are reserved for future use. The third -//! byte indicates the kind of router. Current values are 0 for a server router -//! and 1 for a transit routers. The fourth byte is a hostname length followed -//! directly by a hostname of up to 255 bytes in length. +//! The first byte indicates the version. The second byte is a flags bitfield. +//! The first position `S` indicates a solicitation. The second position `A` +//! indicates an advertisement. The third position `C` indicates DDMv4 +//! (multicast) capability, advertised independently of the version byte so old +//! peers that ignore it still peer at the floor version. All other positions +//! are reserved for future use. The third byte indicates the kind of router. +//! Current values are 0 for a server router and 1 for a transit routers. The +//! fourth byte is a hostname length followed directly by a hostname of up to +//! 255 bytes in length. use thiserror::Error; @@ -104,11 +105,15 @@ mod runtime; #[cfg(all(feature = "backend", target_os = "illumos"))] pub(crate) use runtime::handler; -#[derive(Debug, Copy, Clone)] +// Ordering follows the ascending discriminants, so version-dependent +// behavior can use range checks (`>= Version::V4`) that remain correct +// as newer versions are added. +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] #[repr(u8)] pub enum Version { V2 = 2, V3 = 3, + V4 = 4, } #[derive(Error, Debug)] diff --git a/ddm/src/discovery/runtime.rs b/ddm/src/discovery/runtime.rs index 2cd8c53fc..580eddb9c 100644 --- a/ddm/src/discovery/runtime.rs +++ b/ddm/src/discovery/runtime.rs @@ -29,6 +29,11 @@ const DDM_MADDR: Ipv6Addr = Ipv6Addr::new(0xff02, 0, 0, 0, 0, 0, 0, 0xdd); const DDM_PORT: u16 = 0xddd; const SOLICIT: u8 = 1; const ADVERTISE: u8 = 1 << 1; +// Advertises DDMv4 (multicast) capability without raising the discovery version +// byte past what old peers accept (the byte stays at the V2 floor). Following +// RFC 5492, peers ignore capability bits they do not understand, so this is +// backward compatible. +const MCAST_CAPABLE: u8 = 1 << 2; /// Reinterpret an initialized prefix of `[MaybeUninit]` as `[u8]`. /// @@ -51,7 +56,7 @@ impl DiscoveryPacket { fn new_solicitation(hostname: String, kind: RouterKind) -> Self { Self { version: Version::V2 as u8, - flags: SOLICIT, + flags: SOLICIT | MCAST_CAPABLE, hostname, kind, } @@ -59,7 +64,7 @@ impl DiscoveryPacket { fn new_advertisement(hostname: String, kind: RouterKind) -> Self { Self { version: Version::V2 as u8, - flags: ADVERTISE, + flags: ADVERTISE | MCAST_CAPABLE, hostname, kind, } @@ -89,6 +94,9 @@ struct Neighbor { hostname: String, kind: RouterKind, last_seen: Instant, + /// Last negotiated exchange version, tracked so a capability change on an + /// otherwise stable peer re-emits a neighbor update. + version: Version, } pub(crate) fn handler( @@ -324,6 +332,7 @@ fn handle_msg( msg.hostname, msg.kind, msg.version, + msg.flags, stats, ); } @@ -343,12 +352,53 @@ fn handle_solicitation( } } +/// The outcome of negotiating the session version from a peer's advertisement. +#[derive(Debug, PartialEq)] +enum NegotiatedVersion { + /// A usable version was negotiated directly from the advertised byte and + /// capability flags. + Use(Version), + /// The advertised byte was outside the range this node speaks; the + /// advertisement is rejected and must not replace a valid neighbor. + Rejected, +} + +/// Negotiate the session version from a peer's advertised version byte and +/// capability flags. +/// +/// The version byte is a backward-compatible floor ([`Version::V2`]) that all +/// deployed peers accept. Capability rides flags rather than the byte: +/// [`MCAST_CAPABLE`] signals [`Version::V4`], so a peer at the floor that sets +/// it negotiates V4 while older peers that ignore the flag still session at +/// the floor. A conforming peer never advertises a byte above the known +/// maximum. +/// +/// A byte outside the known range is therefore a malformed or incompatible +/// peer, not a capability hint, so it is rejected rather than capped. See +/// [RFC 5492] for the capability-negotiation model. +/// +/// [RFC 5492]: https://www.rfc-editor.org/rfc/rfc5492 +fn negotiate_version(version: u8, flags: u8) -> NegotiatedVersion { + let base = match version { + 2 => Version::V2, + 3 => Version::V3, + 4 => Version::V4, + _ => return NegotiatedVersion::Rejected, + }; + if (flags & MCAST_CAPABLE) != 0 { + NegotiatedVersion::Use(Version::V4) + } else { + NegotiatedVersion::Use(base) + } +} + fn handle_advertisement( ctx: &HandlerContext, sender: &Ipv6Addr, hostname: String, kind: RouterKind, version: u8, + flags: u8, stats: &Arc, ) { trc!(&ctx.log, ctx.config.if_name, "advert from {}", &hostname); @@ -356,25 +406,15 @@ fn handle_advertisement( .advertisements_received .fetch_add(1, Ordering::Relaxed); - // TODO: version negotiation - // - // Things currently work because ddm v1 does no version checking at all. - // So ddm v2 speakers can send out discovery packets with the version set to - // 2, and ddm v1 speakers can send out discovery packets with the version - // set to 1, and as long a v2 router speaks version 1 after discovering a v1 - // peer, things will work. However, this will not work for version 3. So we - // need to implement version negotiation. This would also not work for - // changes in the discovery protocol, if we were to have changes there. So - // we need to come up with a general way for both protocols to evolve. - let version = match version { - 2 => Version::V2, - 3 => Version::V3, - x => { + let version = match negotiate_version(version, flags) { + NegotiatedVersion::Use(v) => v, + NegotiatedVersion::Rejected => { err!( ctx.log, ctx.config.if_name, - "unknown protocol version {}, known versions are: 1, 2", - x + "unsupported protocol version {version}, this node supports {}..={}", + Version::V2 as u8, + Version::V4 as u8 ); return; } @@ -387,7 +427,7 @@ fn handle_advertisement( return; } }; - match &mut *guard { + let version_changed = match &mut *guard { Some(nbr) => { if *sender != nbr.addr { inf!( @@ -407,6 +447,9 @@ fn handle_advertisement( stats.peer_address_changes.fetch_add(1, Ordering::Relaxed); } nbr.last_seen = Instant::now(); + let version_changed = nbr.version != version; + nbr.version = version; + version_changed } None => { inf!( @@ -422,8 +465,10 @@ fn handle_advertisement( hostname: hostname.clone(), last_seen: Instant::now(), kind, + version, }); stats.peer_established.fetch_add(1, Ordering::Relaxed); + false } }; drop(guard); @@ -433,7 +478,11 @@ fn handle_advertisement( kind, }; let mut info = lock!(ctx.iface.peer_identity); - if info.as_ref() != Some(&new_peer) { + // Emit when the peer's identity changes or when its negotiated version + // changes. A version change on a stable identity (for example a peer that + // begins advertising MCAST_CAPABLE after a restart) must reach the state + // machine, otherwise the exchange keeps using the prior version. + if info.as_ref() != Some(&new_peer) || version_changed { *info = Some(new_peer); drop(info); emit_nbr_update(ctx, sender, version); @@ -494,3 +543,44 @@ fn advertise( stats.advertisements_sent.fetch_add(1, Ordering::Relaxed); Ok(n) } + +#[cfg(test)] +mod tests { + use super::*; + + // Pins the negotiation contract. + // + // Without the capability flag the byte is the floor and maps straight + // through, MCAST_CAPABLE raises any accepted byte to V4, and out-of-range + // bytes are rejected even with the flag set so a malformed advertisement + // cannot replace a valid neighbor. + #[test] + fn negotiate_version_contract() { + use NegotiatedVersion::{Rejected, Use}; + + let cases = [ + // Byte alone is the floor. + (2u8, 0u8, Use(Version::V2)), + (3, 0, Use(Version::V3)), + (4, 0, Use(Version::V4)), + // Capability flag raises any accepted byte to V4. + (2, MCAST_CAPABLE, Use(Version::V4)), + (3, MCAST_CAPABLE, Use(Version::V4)), + (4, MCAST_CAPABLE, Use(Version::V4)), + // Out-of-range bytes are rejected, flag or not. + (0, 0, Rejected), + (1, 0, Rejected), + (1, MCAST_CAPABLE, Rejected), + (5, 0, Rejected), + (255, MCAST_CAPABLE, Rejected), + ]; + + for (version, flags, expected) in cases { + assert_eq!( + negotiate_version(version, flags), + expected, + "negotiate_version({version}, {flags:#05b})" + ); + } + } +} diff --git a/ddm/src/exchange/mod.rs b/ddm/src/exchange/mod.rs index d9f842ebd..e4f0e04c2 100644 --- a/ddm/src/exchange/mod.rs +++ b/ddm/src/exchange/mod.rs @@ -15,20 +15,27 @@ //! model of a ddm router is defined in the state machine implementation in //! [`crate::sm`]. //! -//! The wire types ([`Update`], [`UnderlayUpdate`], [`TunnelUpdate`], and -//! their versioned counterparts) are platform-agnostic and stay in this -//! module. The runtime helpers that drive the HTTP exchange protocol and -//! program forwarding state live in the [`runtime`] submodule and are -//! illumos-only, since they call into [`crate::sys`] to install routes. +//! The wire types (`Update`, `UnderlayUpdate`, `TunnelUpdate`, +//! `MulticastUpdate`, and their versioned counterparts) live in the +//! [`ddm_protocol`] crate. The runtime helpers that drive the HTTP exchange +//! protocol and program forwarding state live in the `runtime` submodule and +//! are illumos-only, since they call into `crate::sys` to install routes. use thiserror::Error; +#[cfg(any(test, all(feature = "backend", target_os = "illumos")))] +mod reconcile; + #[cfg(all(feature = "backend", target_os = "illumos"))] mod runtime; +#[cfg(all(feature = "backend", target_os = "illumos"))] +pub(crate) use reconcile::reconcile_multicast_withdrawals; + #[cfg(all(feature = "backend", target_os = "illumos"))] pub(crate) use runtime::{ - announce_tunnel, announce_underlay, do_pull, handler, pull, + ExchangeHandle, UpdateMode, announce_multicast, announce_tunnel, + announce_underlay, do_pull_v4, handler, pull, withdraw_multicast, withdraw_tunnel, withdraw_underlay, }; @@ -46,6 +53,9 @@ pub enum ExchangeError { #[error("timeout error: {0}")] Timeout(#[from] tokio::time::error::Elapsed), + #[error("peer returned status {0}")] + Status(hyper::StatusCode), + #[error("json error: {0}")] SerdeJson(#[from] serde_json::Error), } diff --git a/ddm/src/exchange/reconcile.rs b/ddm/src/exchange/reconcile.rs new file mode 100644 index 000000000..cc2e67d6e --- /dev/null +++ b/ddm/src/exchange/reconcile.rs @@ -0,0 +1,262 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Side-effect-free reconciliation management shared by exchange +//! runtime paths, taking state in as arguments and returning updates +//! without database or network effects. + +use ddm_api_types::net::MulticastOrigin; +use ddm_protocol::v4::{ + MulticastPathHop, MulticastPathVector, MulticastUpdate, +}; + +/// Rewrite each withdrawal as a replacement announcement when another path to +/// the origin remains, and as a final withdrawal otherwise. +/// +/// Downstream peers keep exactly one route per `(origin, nexthop)`, with this +/// router as the nexthop, regardless of how many paths line up behind it. +/// Blindly forwarding a withdrawal would drop the peer's only route through +/// this router even when the origin is still reachable in another way. For each +/// withdrawal, this checks the local origins first, then the best entry in +/// the imported set. A remaining path produces an announcement with a +/// refreshed path vector. +/// +/// Only when nothing remains is the final withdrawal emitted with `local_hop` +/// appended. +/// +/// The reachability snapshot always describes post-modification state. Peer +/// expiry and renumber capture it in `remove_nexthop_routes`, exchange +/// updates in `update_imported_mcast`, and the admin withdraws read it at +/// processing time via `multicast_reachability`, which is still +/// post-modification because the event is enqueued only after the +/// modification lands. +pub(crate) fn reconcile_multicast_withdrawals<'a>( + withdrawals: impl IntoIterator, + reachability: &crate::db::MulticastReachability, + local_hop: &MulticastPathHop, +) -> MulticastUpdate { + let mut update = MulticastUpdate::default(); + + for withdrawal in withdrawals { + let replacement = MulticastOrigin::try_from(&withdrawal.origin) + .ok() + .and_then(|origin| { + if let Some(local_origin) = + reachability.originated().get(&origin) + { + return Some(MulticastPathVector { + origin: local_origin.into(), + path: vec![local_hop.clone()], + }); + } + + reachability + .imported() + .iter() + .filter(|route| route.origin == origin) + // Route identity guarantees one entry per nexthop for this + // origin. Address order is only a stable tie-breaker (it + // does not assign semantics to multicast metric). + .min_by_key(|route| route.nexthop) + .map(|route| { + let mut path = route.path.clone(); + path.push(local_hop.clone()); + MulticastPathVector { + origin: (&route.origin).into(), + path, + } + }) + }); + + match replacement { + Some(replacement) => { + update.announce.insert(replacement); + } + // A degraded snapshot cannot confirm the absence of a local + // origin, so a final withdrawal here could tear down a route to + // an origin that is still reachable. Dropping the withdrawal is + // the safe direction. The worst case is a transient stale route + // to an origin that really is gone, while a false withdrawal + // could remove a peer's only path through this router. The + // periodic exchange resync repairs the drift either way. + None if reachability.origins_degraded() => {} + None => { + update + .withdraw + .insert(withdrawal.with_hop(local_hop.clone())); + } + } + } + + update +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::{Db, MulticastReachability}; + use ddm_api_types::db::MulticastRoute; + use slog::Logger; + use std::collections::HashSet; + use std::net::Ipv6Addr; + use tempfile::TempDir; + + fn origin(metric: u64) -> MulticastOrigin { + serde_json::from_value(serde_json::json!({ + "overlay_group": "233.252.0.1", + "underlay_group": "ff04::1", + "vni": 77, + "metric": metric, + })) + .unwrap() + } + + fn hop(router_id: &str, last: u16) -> MulticastPathHop { + MulticastPathHop::new( + router_id.to_string(), + Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, last), + ) + } + + /// Build a `MulticastReachability` snapshot through `Db` persistence + /// rather than constructing it by hand, so tests exercise the same + /// snapshot path production uses. Any origins in `originated` are + /// persisted before the imported set is applied, so the captured + /// snapshot reflects both sources of reachability. + fn snapshot( + imported: HashSet, + originated: HashSet, + ) -> (TempDir, MulticastReachability) { + let dir = TempDir::new().unwrap(); + let log = Logger::root(slog::Discard, slog::o!()); + let db = Db::new(dir.path().to_str().unwrap(), log).unwrap(); + if !originated.is_empty() { + db.originate_mcast(&originated).unwrap(); + } + let (_delta, reachability) = db + .update_imported_mcast_with_reachability( + &imported, + &HashSet::new(), + ); + (dir, reachability) + } + + #[test] + fn remaining_import_replaces_withdrawal_and_refreshes_path() { + let origin = origin(10); + let withdrawn = MulticastPathVector { + origin: (&origin).into(), + path: vec![hop("withdrawn", 1)], + }; + let remaining_hop = hop("remaining", 2); + let imported = HashSet::from([MulticastRoute { + origin: origin.clone(), + nexthop: "fe80::2".parse().unwrap(), + path: vec![remaining_hop.clone()], + }]); + let local_hop = hop("local", 3); + let (_dir, remaining) = snapshot(imported, HashSet::new()); + + let update = reconcile_multicast_withdrawals( + [&withdrawn], + &remaining, + &local_hop, + ); + + assert!(update.withdraw.is_empty()); + let replacement = update.announce.iter().next().unwrap(); + assert_eq!(replacement.path, vec![remaining_hop, local_hop]); + } + + #[test] + fn remaining_local_origin_replaces_withdrawal() { + let origin = origin(10); + let withdrawn = MulticastPathVector { + origin: (&origin).into(), + path: vec![hop("withdrawn", 1)], + }; + let local_hop = hop("local", 3); + let (_dir, remaining) = + snapshot(HashSet::new(), HashSet::from([origin])); + + let update = reconcile_multicast_withdrawals( + [&withdrawn], + &remaining, + &local_hop, + ); + + assert!(update.withdraw.is_empty()); + let replacement = update.announce.iter().next().unwrap(); + assert_eq!(replacement.path, vec![local_hop]); + } + + #[test] + fn final_withdrawal_preserves_path_and_appends_local_hop() { + let origin = origin(10); + let withdrawn_hop = hop("withdrawn", 1); + let withdrawn = MulticastPathVector { + origin: (&origin).into(), + path: vec![withdrawn_hop.clone()], + }; + let local_hop = hop("local", 3); + let (_dir, remaining) = snapshot(HashSet::new(), HashSet::new()); + + let update = reconcile_multicast_withdrawals( + [&withdrawn], + &remaining, + &local_hop, + ); + + assert!(update.announce.is_empty()); + let forwarded = update.withdraw.iter().next().unwrap(); + assert_eq!(forwarded.path, vec![withdrawn_hop, local_hop]); + } + + /// The processing-time read path used by admin withdraw + /// revalidation must observe both persisted origins and the current + /// imported set. + #[test] + fn multicast_reachability_reads_current_imported_and_originated() { + let dir = TempDir::new().unwrap(); + let log = Logger::root(slog::Discard, slog::o!()); + let db = Db::new(dir.path().to_str().unwrap(), log).unwrap(); + + // Distinct overlay/underlay groups so identity-based equality on + // `MulticastOrigin` keeps the persisted and imported origins apart. + let local_origin: MulticastOrigin = + serde_json::from_value(serde_json::json!({ + "overlay_group": "233.252.0.1", + "underlay_group": "ff04::1", + "vni": 77, + "metric": 10, + })) + .unwrap(); + + let imported_origin: MulticastOrigin = + serde_json::from_value(serde_json::json!({ + "overlay_group": "233.252.0.2", + "underlay_group": "ff04::2", + "vni": 77, + "metric": 20, + })) + .unwrap(); + + let route = MulticastRoute { + origin: imported_origin, + nexthop: "fe80::2".parse().unwrap(), + path: vec![hop("remote", 2)], + }; + + db.originate_mcast(&HashSet::from([local_origin.clone()])) + .unwrap(); + db.update_imported_mcast( + &HashSet::from([route.clone()]), + &HashSet::new(), + ); + + let reachability = db.multicast_reachability(); + assert_eq!(reachability.imported(), &HashSet::from([route])); + assert_eq!(reachability.originated(), &HashSet::from([local_origin])); + } +} diff --git a/ddm/src/exchange/runtime.rs b/ddm/src/exchange/runtime.rs index 653824e43..b36027186 100644 --- a/ddm/src/exchange/runtime.rs +++ b/ddm/src/exchange/runtime.rs @@ -7,12 +7,18 @@ //! plumbing that drains received updates into the local DB and the //! forwarding platform via [`crate::sys`]. illumos-only. -use super::ExchangeError; +use super::{ExchangeError, reconcile_multicast_withdrawals}; use crate::db::{Route, effective_route_set}; use crate::discovery::Version; use crate::sm::{Config, Event, PeerEvent, SmContext}; -use crate::{dbg, err, inf, wrn}; -use ddm_api_types::db::{RouterKind, TunnelRoute}; +use crate::{dbg, err, inf, trc, wrn}; +use ddm_api_types::db::{MulticastRoute, RouterKind, TunnelRoute}; +use ddm_api_types::net::MulticastOrigin; +use ddm_protocol::v3::{PathVector, TunnelOrigin}; +use ddm_protocol::v4::{ + MulticastPathHop, MulticastPathVector, MulticastUpdate, PullResponse, + Update, +}; use ddm_protocol::{v2, v3}; use dropshot::ApiDescription; use dropshot::ConfigDropshot; @@ -29,6 +35,7 @@ use http_body_util::BodyExt; use hyper::body::Bytes; use hyper_util::client::legacy::Client; use hyper_util::rt::TokioExecutor; +use mg_common::lock; use slog::{Logger, o}; use std::collections::HashSet; use std::net::{Ipv6Addr, SocketAddrV6}; @@ -40,6 +47,11 @@ use tokio::time::timeout; const UNIT_EXCHANGE_SERVER: &str = "exchange_server"; +/// Bound on an entire pull request, from dispatch through to reading the full +/// response. Pulls run on state machine threads, so a stalled peer must not +/// block event handling. +const PULL_TIMEOUT: Duration = Duration::from_millis(250); + #[derive(Clone)] pub struct HandlerContext { ctx: SmContext, @@ -47,6 +59,46 @@ pub struct HandlerContext { log: Logger, } +/// How an update's imported routes propagate beyond the local DB. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum UpdateMode { + /// Import into the local DB only. + ImportOnly, + /// Import and re-announce to this router's other peers. Only transit + /// routers act on this. Server routers treat it as [`Self::ImportOnly`]. + Redistribute, +} + +/// A handle to a running exchange server, pairing the server task with the +/// shared request context so the state machine can rebind the peer address +/// on renumber without restarting the server. +/// +/// A renumber occurs when a peer's link-local unicast address changes. +/// [`crate::discovery`] detects the change and re-advertises the neighbor +/// under the new address. The neighbor is still the same router, so the +/// exchange server keeps running and only the nexthop address it assigns to +/// imports changes. +pub struct ExchangeHandle { + thread: tokio::task::JoinHandle<()>, + context: Arc>, +} + +impl ExchangeHandle { + pub fn abort(&self) { + self.thread.abort(); + } + + /// Rebind the handler's peer address after a renumber. The handler + /// assigns this address as the nexthop on every route it imports, so it + /// must track the state machine's view of the peer or else, post-renumber + /// imports leak under the prior address. + pub fn renumber_peer(&self, peer: Ipv6Addr) { + // Safe to block: callers run on state machine threads, outside + // the runtime. + self.context.blocking_lock().peer = peer; + } +} + pub(crate) fn announce_underlay( ctx: &SmContext, config: Config, @@ -99,15 +151,52 @@ pub(crate) fn withdraw_tunnel( send_update(ctx, config, update.into(), addr, version, rt, log) } -pub(crate) fn do_pull( +pub(crate) fn announce_multicast( + ctx: &SmContext, + config: Config, + groups: HashSet, + addr: Ipv6Addr, + version: Version, + rt: Arc, + log: Logger, +) -> Result<(), ExchangeError> { + let update = MulticastUpdate::announce(groups); + send_update(ctx, config, update.into(), addr, version, rt, log) +} + +pub(crate) fn withdraw_multicast( + ctx: &SmContext, + config: Config, + groups: HashSet, + addr: Ipv6Addr, + version: Version, + rt: Arc, + log: Logger, +) -> Result<(), ExchangeError> { + let update = MulticastUpdate::withdraw(groups); + send_update(ctx, config, update.into(), addr, version, rt, log) +} + +pub(crate) fn do_pull_v4( + ctx: &SmContext, + addr: &Ipv6Addr, + rt: &Arc, +) -> Result { + let if_index = ctx.config.if_index; + let port = ctx.config.exchange_port; + let uri = format!("http://[{addr}%{if_index}]:{port}/v4/pull"); + let body = do_pull_common(uri, rt)?; + Ok(serde_json::from_slice(&body)?) +} + +pub(crate) fn do_pull_v3( ctx: &SmContext, addr: &Ipv6Addr, rt: &Arc, ) -> Result { - let uri = format!( - "http://[{}%{}]:{}/v3/pull", - addr, ctx.config.if_index, ctx.config.exchange_port, - ); + let if_index = ctx.config.if_index; + let port = ctx.config.exchange_port; + let uri = format!("http://[{addr}%{if_index}]:{port}/v3/pull"); let body = do_pull_common(uri, rt)?; Ok(serde_json::from_slice(&body)?) } @@ -125,6 +214,21 @@ pub(crate) fn do_pull_v2( Ok(serde_json::from_slice(&body)?) } +fn require_success( + response: hyper::Response, +) -> Result, ExchangeError> { + if response.status().is_success() { + Ok(response) + } else { + Err(ExchangeError::Status(response.status())) + } +} + +/// Fetch a pull response body, accepting only successful HTTP responses. +/// +/// The status is checked before the body reaches a versioned decoder. This is +/// especially important for V4, whose optional response fields could otherwise +/// make a Dropshot JSON error look like an empty route set. fn do_pull_common( uri: String, rt: &Arc, @@ -139,37 +243,104 @@ fn do_pull_common( let resp = client.request(req); + // The timeout covers reading the body too, since a peer that stalls + // mid-response would otherwise block indefinitely. rt.block_on(async move { - let body = timeout(Duration::from_millis(250), resp) - .await?? - .into_body() - .collect() - .await? - .to_bytes(); - Ok(body) + timeout(PULL_TIMEOUT, async { + let resp = require_success(resp.await?)?; + Ok(resp.into_body().collect().await?.to_bytes()) + }) + .await? }) } +/// Pull the peer's routes and import them. +/// +/// When `mode` is [`UpdateMode::Redistribute`] and this router is a transit, +/// the imported set is also announced to the other peers. The initial pull on +/// entering the exchange state redistributes. The periodic pull imports only. +/// Each router runs its own periodic pull, so a route learned here still +/// reaches every router through that router's pull. Redistributing on every +/// cycle would only resend updates transit peers already hold in steady +/// state. +/// +/// A non-successful HTTP response aborts the pull before decoding or +/// reconciliation, leaving the routes previously imported from that peer +/// unchanged. pub(crate) fn pull( ctx: SmContext, addr: Ipv6Addr, version: Version, rt: Arc, log: Logger, + mode: UpdateMode, ) -> Result<(), ExchangeError> { - let pr: v3::PullResponse = match version { - Version::V2 => do_pull_v2(&ctx, &addr, &rt)?.into(), - Version::V3 => do_pull(&ctx, &addr, &rt)?, + let pr: PullResponse = match version { + Version::V2 => { + v3::PullResponse::from(do_pull_v2(&ctx, &addr, &rt)?).into() + } + Version::V3 => do_pull_v3(&ctx, &addr, &rt)?.into(), + Version::V4 => do_pull_v4(&ctx, &addr, &rt)?, }; - let update = v3::Update::announce(pr); + // A multicast-capable peer's pull response carries its complete + // advertisable multicast set, so an imported multicast route from this + // peer that is absent from the response indicates a withdraw we missed. + // Therefore, we synthesize withdraws for the absentee vectors so that the + // periodic pull repairs subtractive as well as additive drift. + // + // Underlay and tunnel imports keep their push-only withdraw semantics. + // Multicast reconciliation matters more because a stale import holds a + // DPD replication member. Reconciliation applies only to peers that + // negotiated wire protocol version 4 or later, the first version to + // carry multicast. An earlier response has no multicast half, and its + // converted empty set must not be read as a full withdraw. + let mcast_withdraw: HashSet = if version < Version::V4 + { + HashSet::new() + } else { + let announced: HashSet = pr + .multicast + .iter() + .flatten() + .filter_map(|pv| MulticastOrigin::try_from(&pv.origin).ok()) + .collect(); + ctx.db + .imported_mcast() + .iter() + .filter(|route| { + route.nexthop == addr && !announced.contains(&route.origin) + }) + .map(|route| MulticastPathVector { + origin: (&route.origin).into(), + path: Vec::new(), + }) + .collect() + }; - let hctx = HandlerContext { + let mut update = Update::announce(pr); + if !mcast_withdraw.is_empty() { + dbg!( + log, + ctx.config.if_name, + "pull reconcile: withdrawing {} stale multicast routes", + mcast_withdraw.len(), + ); + match update.multicast.as_mut() { + Some(m) => m.withdraw = mcast_withdraw, + None => { + update.multicast = + Some(MulticastUpdate::withdraw(mcast_withdraw)) + } + } + } + + let handler = HandlerContext { ctx, peer: addr, - log: log.clone(), + log, }; - handle_update(&update, &hctx); + handle_update(&update, &handler, mode); Ok(()) } @@ -177,50 +348,39 @@ pub(crate) fn pull( fn send_update( ctx: &SmContext, config: Config, - update: v3::Update, + update: Update, addr: Ipv6Addr, version: Version, rt: Arc, log: Logger, ) -> Result<(), ExchangeError> { - ctx.stats.updates_sent.fetch_add(1, Ordering::Relaxed); - match version { + // The update arrives in the latest wire form. Downconvert through + // consecutive versions when a peer negotiated an older protocol. + // Conversion drops content the peer's version cannot represent (multicast + // did not exist before V4, for example), so the downconverted form can + // be empty. We skip the send in that case rather than emit an empty + // payload. + let (payload, path) = match version { Version::V2 => { - send_update_v2(ctx, config, update.into(), addr, rt, log) + let update = v2::Update::from(v3::Update::from(update)); + if update.underlay.is_none() && update.tunnel.is_none() { + return Ok(()); + } + (serde_json::to_string(&update)?, "v2") } - Version::V3 => send_update_v3(ctx, config, update, addr, rt, log), - } -} - -fn send_update_v2( - ctx: &SmContext, - config: Config, - update: v2::Update, - addr: Ipv6Addr, - rt: Arc, - log: Logger, -) -> Result<(), ExchangeError> { - let payload = serde_json::to_string(&update)?; - let uri = format!( - "http://[{}%{}]:{}/v2/push", - addr, config.if_index, config.exchange_port, - ); - send_update_common(ctx, uri, payload, config, rt, log) -} - -fn send_update_v3( - ctx: &SmContext, - config: Config, - update: v3::Update, - addr: Ipv6Addr, - rt: Arc, - log: Logger, -) -> Result<(), ExchangeError> { - let payload = serde_json::to_string(&update)?; - let uri = format!( - "http://[{}%{}]:{}/v3/push", - addr, config.if_index, config.exchange_port, - ); + Version::V3 => { + let update = v3::Update::from(update); + if update.underlay.is_none() && update.tunnel.is_none() { + return Ok(()); + } + (serde_json::to_string(&update)?, "v3") + } + Version::V4 => (serde_json::to_string(&update)?, "v4"), + }; + ctx.stats.updates_sent.fetch_add(1, Ordering::Relaxed); + let if_index = config.if_index; + let port = config.exchange_port; + let uri = format!("http://[{addr}%{if_index}]:{port}/{path}/push"); send_update_common(ctx, uri, payload, config, rt, log) } @@ -243,23 +403,23 @@ fn send_update_common( let resp = client.request(req); + // A completed request only counts as delivered when the peer's handler + // reported success. Connection failures and error statuses must surface as + // errors so the state machine can expire the peer. rt.block_on(async move { - match timeout(Duration::from_millis(config.exchange_timeout), resp) - .await - { - Ok(_) => Ok(()), - Err(e) => { - err!( - log, - config.if_name, - "peer request timeout to {}: {}", - uri, - e, - ); - ctx.stats.update_send_fail.fetch_add(1, Ordering::Relaxed); - Err(e.into()) - } + let result: Result<(), ExchangeError> = async { + let resp = + timeout(Duration::from_millis(config.exchange_timeout), resp) + .await??; + require_success(resp)?; + Ok(()) } + .await; + if let Err(e) = &result { + err!(log, config.if_name, "peer update to {uri} failed: {e}"); + ctx.stats.update_send_fail.fetch_add(1, Ordering::Relaxed); + } + result }) } @@ -268,12 +428,13 @@ pub fn handler( addr: Ipv6Addr, peer: Ipv6Addr, log: Logger, -) -> Result, String> { +) -> Result { let context = Arc::new(Mutex::new(HandlerContext { ctx: ctx.clone(), log: log.clone(), peer, })); + let handler_ctx = Arc::clone(&context); let sa = SocketAddrV6::new(addr, ctx.config.exchange_port, 0, 0); @@ -307,7 +468,7 @@ pub fn handler( } })?; - Ok(ctx.rt.spawn(async move { + let thread = ctx.rt.spawn(async move { match server.start().await { Ok(_) => wrn!( log, @@ -321,7 +482,12 @@ pub fn handler( e ), } - })) + }); + + Ok(ExchangeHandle { + thread, + context: handler_ctx, + }) } pub fn api_description() -> Result< @@ -330,9 +496,11 @@ pub fn api_description() -> Result< > { let mut api = ApiDescription::new(); api.register(push_handler_v2)?; - api.register(push_handler)?; + api.register(push_handler_v3)?; + api.register(push_handler_v4)?; api.register(pull_handler_v2)?; - api.register(pull_handler)?; + api.register(pull_handler_v3)?; + api.register(pull_handler_v4)?; Ok(api) } @@ -345,17 +513,23 @@ async fn push_handler_v2( request: TypedBody, ) -> Result { let update_v2 = request.into_inner(); - let update = v3::Update::from(update_v2); + let update = Update::from(v3::Update::from(update_v2)); push_handler_common(ctx, update).await } -#[endpoint { - method = PUT, - path = "/v3/push", -}] -async fn push_handler( +#[endpoint { method = PUT, path = "/v3/push" }] +async fn push_handler_v3( ctx: RequestContext>>, request: TypedBody, +) -> Result { + let update = Update::from(request.into_inner()); + push_handler_common(ctx, update).await +} + +#[endpoint { method = PUT, path = "/v4/push" }] +async fn push_handler_v4( + ctx: RequestContext>>, + request: TypedBody, ) -> Result { let update = request.into_inner(); push_handler_common(ctx, update).await @@ -363,11 +537,11 @@ async fn push_handler( async fn push_handler_common( ctx: RequestContext>>, - update: v3::Update, + update: Update, ) -> Result { let ctx = ctx.context().lock().await.clone(); tokio::task::spawn_blocking(move || { - handle_update(&update, &ctx); + handle_update(&update, &ctx, UpdateMode::Redistribute); }) .await .map_err(|e| { @@ -396,19 +570,18 @@ async fn pull_handler_v2( if route.nexthop == ctx.peer { continue; } - let mut pv = v3::PathVector { + let mut path_vector = PathVector { destination: route.destination, path: route.path.clone(), }; - pv.path.push(ctx.ctx.hostname.clone()); - underlay.insert(pv); + path_vector.path.push(ctx.ctx.hostname.clone()); + underlay.insert(path_vector); } for route in &ctx.ctx.db.imported_tunnel() { if route.nexthop == ctx.peer { continue; } - let tv = route.origin; - tunnel.insert(tv); + tunnel.insert(route.origin); } } let originated = ctx @@ -417,11 +590,11 @@ async fn pull_handler_v2( .originated() .map_err(|e| HttpError::for_internal_error(e.to_string()))?; for prefix in &originated { - let pv = v3::PathVector { + let path_vector = PathVector { destination: *prefix, path: vec![ctx.ctx.hostname.clone()], }; - underlay.insert(pv); + underlay.insert(path_vector); } let originated_tunnel = ctx @@ -453,51 +626,43 @@ async fn pull_handler_v2( })) } -#[endpoint { - method = GET, - path = "/v3/pull", -}] -async fn pull_handler( - ctx: RequestContext>>, -) -> Result, HttpError> { - let ctx = ctx.context().lock().await.clone(); - +/// Collect underlay and tunnel routes for pull responses (shared by V3/V4). +fn collect_underlay_tunnel( + ctx: &HandlerContext, +) -> Result<(HashSet, HashSet), HttpError> { let mut underlay = HashSet::new(); let mut tunnel = HashSet::new(); - // Only transit routers redistribute prefixes if ctx.ctx.config.kind == RouterKind::Transit { for route in &ctx.ctx.db.imported() { - // don't redistribute prefixes to their originators if route.nexthop == ctx.peer { continue; } - let mut pv = v3::PathVector { + let mut path_vector = PathVector { destination: route.destination, path: route.path.clone(), }; - pv.path.push(ctx.ctx.hostname.clone()); - underlay.insert(pv); + path_vector.path.push(ctx.ctx.hostname.clone()); + underlay.insert(path_vector); } for route in &ctx.ctx.db.imported_tunnel() { if route.nexthop == ctx.peer { continue; } - let tv = route.origin; - tunnel.insert(tv); + tunnel.insert(route.origin); } } + let originated = ctx .ctx .db .originated() .map_err(|e| HttpError::for_internal_error(e.to_string()))?; for prefix in &originated { - let pv = v3::PathVector { + underlay.insert(PathVector { destination: *prefix, path: vec![ctx.ctx.hostname.clone()], - }; - underlay.insert(pv); + }); } let originated_tunnel = ctx @@ -506,35 +671,112 @@ async fn pull_handler( .originated_tunnel() .map_err(|e| HttpError::for_internal_error(e.to_string()))?; for prefix in &originated_tunnel { - let tv = v3::TunnelOrigin { + tunnel.insert(TunnelOrigin { overlay_prefix: prefix.overlay_prefix, boundary_addr: prefix.boundary_addr, vni: prefix.vni, metric: prefix.metric, - }; - tunnel.insert(tv); + }); + } + + Ok((underlay, tunnel)) +} + +/// Collect multicast routes for V4 pull responses. +fn collect_multicast( + ctx: &HandlerContext, +) -> Result, HttpError> { + let mut multicast = HashSet::new(); + + if ctx.ctx.config.kind == RouterKind::Transit { + for route in &ctx.ctx.db.imported_mcast() { + if route.nexthop == ctx.peer { + continue; + } + let hop = MulticastPathHop::new( + ctx.ctx.hostname.clone(), + ctx.ctx.config.addr, + ); + let mut path = route.path.clone(); + path.push(hop); + multicast.insert(MulticastPathVector { + origin: (&route.origin).into(), + path, + }); + } + } + + let originated_mcast = ctx + .ctx + .db + .originated_mcast() + .map_err(|e| HttpError::for_internal_error(e.to_string()))?; + for origin in &originated_mcast { + let hop = MulticastPathHop::new( + ctx.ctx.hostname.clone(), + ctx.ctx.config.addr, + ); + multicast.insert(MulticastPathVector { + origin: origin.into(), + path: vec![hop], + }); } + Ok(multicast) +} + +#[endpoint { method = GET, path = "/v3/pull" }] +async fn pull_handler_v3( + ctx: RequestContext>>, +) -> Result, HttpError> { + let ctx = ctx.context().lock().await.clone(); + let (underlay, tunnel) = collect_underlay_tunnel(&ctx)?; Ok(HttpResponseOk(v3::PullResponse { - underlay: if underlay.is_empty() { - None - } else { - Some(underlay) - }, - tunnel: if tunnel.is_empty() { - None - } else { - Some(tunnel) - }, + underlay: crate::non_empty(underlay), + tunnel: crate::non_empty(tunnel), })) } -fn handle_update(update: &v3::Update, ctx: &HandlerContext) { +#[endpoint { method = GET, path = "/v4/pull" }] +async fn pull_handler_v4( + ctx: RequestContext>>, +) -> Result, HttpError> { + let ctx = ctx.context().lock().await.clone(); + let (underlay, tunnel) = collect_underlay_tunnel(&ctx)?; + let multicast = collect_multicast(&ctx)?; + Ok(HttpResponseOk(PullResponse { + underlay: crate::non_empty(underlay), + tunnel: crate::non_empty(tunnel), + multicast: crate::non_empty(multicast), + })) +} + +fn handle_update(update: &Update, ctx: &HandlerContext, mode: UpdateMode) { ctx.ctx .stats .updates_received .fetch_add(1, Ordering::Relaxed); + // Route application and peer cleanup take the same per-interface lock. + // This lets discovery publish identity and liveness changes without + // waiting for DPD, OPTE, or datastore work below. Once the lock is held, + // a brief identity check rejects an update whose peer has already expired + // or renumbered. Otherwise, the subsequent cleanup waits and removes + // anything this update imports. + let _route_update = lock!(ctx.ctx.iface.route_update); + let current_peer = lock!(ctx.ctx.iface.peer_identity) + .as_ref() + .map(|peer| peer.addr); + if current_peer != Some(ctx.peer) { + inf!( + ctx.log, + ctx.ctx.config.if_name, + "discarding update from stale peer {}", + ctx.peer, + ); + return; + } + if let Some(underlay_update) = &update.underlay { handle_underlay_update(underlay_update, ctx); } @@ -543,9 +785,25 @@ fn handle_update(update: &v3::Update, ctx: &HandlerContext) { handle_tunnel_update(tunnel_update, ctx); } - // distribute updates - - if ctx.ctx.config.kind == RouterKind::Transit { + // Only transit routers redistribute, so demote the mode on a server + // before it reaches the multicast handler. Only the redistribution path + // reconciles against a reachability snapshot, so only it pays for + // capturing one. + let mode = if ctx.ctx.config.kind == RouterKind::Transit { + mode + } else { + UpdateMode::ImportOnly + }; + let mcast_reachability = update + .multicast + .as_ref() + .and_then(|mu| handle_multicast_update(mu, ctx, mode)); + + // Event delivery from different interfaces is intentionally not globally + // ordered. A reversed pair can expose an older multicast view until the + // next successful V4 pull, whose complete response repairs missing and + // stale imports. This avoids a global lock on every multicast change. + if mode == UpdateMode::Redistribute { dbg!( ctx.log, ctx.ctx.config.if_name, @@ -558,13 +816,69 @@ fn handle_update(update: &v3::Update, ctx: &HandlerContext) { .as_ref() .map(|update| update.with_path_element(ctx.ctx.hostname.clone())); - let push = v3::Update { + // Multicast loop prevention is asymmetric with the underlay. The + // underlay filters on send, skipping any route whose nexthop is the + // destination peer. Multicast drops, on receipt, any + // announcement whose path already carries our router_id. The path + // check is required because a replacement announcement goes to every + // peer, so a peer can appear mid-path rather than as the nexthop, + // and paths can cross several transits, forming loops longer than + // the immediate echo. + // + // The same filter applies here before redistributing. A peer already + // in a vector's path would drop it anyway, but a peer that is not + // would import a looped path. + let hostname = &ctx.ctx.hostname; + + // The snapshot came from the `handle_multicast_update` modification, so + // reconciliation reads state consistent with the local application. + let multicast = + update + .multicast + .as_ref() + .zip(mcast_reachability.as_ref()) + .map(|(update, reachability)| { + let hop = MulticastPathHop::new( + hostname.clone(), + ctx.ctx.config.addr, + ); + + let is_loop_free = |path_vector: &&MulticastPathVector| { + !path_vector + .path + .iter() + .any(|hop| &hop.router_id == hostname) + }; + + let mut reconciled = reconcile_multicast_withdrawals( + update.withdraw.iter().filter(is_loop_free), + reachability, + &hop, + ); + reconciled.announce.extend( + update.announce.iter().filter(is_loop_free).map( + |path_vector| path_vector.with_hop(hop.clone()), + ), + ); + reconciled + }); + + let push = Arc::new(Update { underlay, tunnel: update.tunnel.clone(), - }; + multicast, + }); for ec in &ctx.ctx.event_channels { - ec.send(Event::Peer(PeerEvent::Push(push.clone()))).unwrap(); + if let Err(e) = + ec.send(Event::Peer(PeerEvent::Push(Arc::clone(&push)))) + { + err!( + ctx.log, + ctx.ctx.config.if_name, + "deliver redistributed update: {e}", + ); + } } } } @@ -711,3 +1025,136 @@ fn handle_underlay_update(update: &v3::UnderlayUpdate, ctx: &HandlerContext) { .imported_underlay_prefixes .store(ctx.ctx.db.imported_count() as u64, Ordering::Relaxed); } + +fn handle_multicast_update( + update: &MulticastUpdate, + ctx: &HandlerContext, + mode: UpdateMode, +) -> Option { + let db = &ctx.ctx.db; + let hostname = &ctx.ctx.hostname; + + let mut import = HashSet::new(); + let mut remove = HashSet::new(); + // A replacement is broadcast to every peer, including peers already in + // its path. For such a peer, the looped announce implicitly invalidates + // its old route through the sender. A clean vector for the same + // `(origin, peer)` in this update takes precedence. + for path_vector in &update.announce { + // Promote the wire origin to the validated form. Peer-supplied routes + // are otherwise trusted, but the underlay group reaches DPD directly, + // so a promotion enforces its ff04::/64 invariant (and the VNI range) + // before the route can be stored. An invalid origin is dropped rather + // than tracking a group DPD would refuse to program. + let origin = match MulticastOrigin::try_from(&path_vector.origin) { + Ok(origin) => origin, + Err(e) => { + wrn!( + ctx.log, + ctx.ctx.config.if_name, + "dropping multicast announce for {}; {e}", + path_vector.origin.overlay_group, + ); + continue; + } + }; + + let route = MulticastRoute { + origin, + nexthop: ctx.peer, + path: path_vector.path.clone(), + }; + if path_vector + .path + .iter() + .any(|hop| &hop.router_id == hostname) + { + if !import.contains(&route) { + dbg!( + ctx.log, + ctx.ctx.config.if_name, + "removing multicast route for {} via {}; \ + looped announce (path length {})", + path_vector.origin.overlay_group, + ctx.peer, + path_vector.path.len(), + ); + remove.insert(route); + } + } else { + // This also cancels an implicit removal if a looped vector for + // the same route appeared earlier in the unordered announce set. + remove.remove(&route); + import.insert(route); + } + } + + for path_vector in &update.withdraw { + // A withdrawal whose path already contains this router is an echo of + // a local redistribution and must not be acted on. + if path_vector + .path + .iter() + .any(|hop| &hop.router_id == hostname) + { + trc!( + ctx.log, + ctx.ctx.config.if_name, + "dropping multicast withdraw for {}; loop detected \ + (path length {})", + path_vector.origin.overlay_group, + path_vector.path.len(), + ); + continue; + } + + // A withdraw carrying an invalid origin cannot match a stored route, + // since storage only admits promoted origins, so drop it here too. + let origin = match MulticastOrigin::try_from(&path_vector.origin) { + Ok(origin) => origin, + Err(e) => { + wrn!( + ctx.log, + ctx.ctx.config.if_name, + "dropping multicast withdraw for {}; {e}", + path_vector.origin.overlay_group, + ); + continue; + } + }; + + // Route identity is (origin, nexthop), so an empty path matches. + remove.insert(MulticastRoute { + origin, + nexthop: ctx.peer, + path: Vec::new(), + }); + } + + // Atomic import + delete + diff under a single lock. The redistribution + // path also reconciles against a post-modification reachability snapshot, + // captured under that same lock scope. + let (delta, reachability) = match mode { + UpdateMode::Redistribute => { + let (delta, reachability) = + db.update_imported_mcast_with_reachability(&import, &remove); + (delta, Some(reachability)) + } + UpdateMode::ImportOnly => { + (db.update_imported_mcast(&import, &remove), None) + } + }; + + // Notify the multicast sweep of each affected underlay group so it + // reconciles the group's DPD members. Only the sweep writes to DPD. + // + // This handler records the import and signals, deriving the notification + // from the effective diff rather than the requested sets avoids waking the + // sweep for routes that were already present or already absent. + crate::mcast::notify_affected_groups( + delta.added.iter().chain(delta.removed.iter()), + &ctx.ctx.mcast_notify, + ); + + reachability +} diff --git a/ddm/src/lib.rs b/ddm/src/lib.rs index 6a5e2a68a..2bfb631e9 100644 --- a/ddm/src/lib.rs +++ b/ddm/src/lib.rs @@ -6,6 +6,8 @@ pub mod admin; pub mod db; pub mod discovery; pub mod exchange; +#[cfg(feature = "backend")] +pub mod mcast; pub mod oxstats; pub mod sm; #[cfg(all(feature = "backend", target_os = "illumos"))] @@ -15,6 +17,29 @@ pub const COMPONENT_DDM: &str = "ddm"; pub const MOD_ADMIN: &str = "admin"; pub const MOD_EXCHANGE: &str = "exchange"; +/// Capacity of the channel carrying wake hints to the multicast sweep. +/// +/// Wake messages are best-effort: senders drop a hint when the channel is full, +/// and the sweep's periodic reconciliation of its full tracked set repairs the +/// omission. A depth of one would suffice semantically, while this small buffer +/// absorbs routine bursts. +/// +/// This constant lives outside the `backend`-feature-gated `mcast` module +/// because `ddmd` wires the channel in every supported feature configuration. +pub const MCAST_NOTIFY_CHANNEL_DEPTH: usize = 8; + +/// Wrap a set in `Some`, treating an empty set as absence. +/// +/// # Returns +/// +/// `None` if `set` is empty, otherwise `Some(set)`. +#[cfg(all(feature = "backend", target_os = "illumos"))] +pub(crate) fn non_empty( + set: std::collections::HashSet, +) -> Option> { + (!set.is_empty()).then_some(set) +} + #[macro_export] macro_rules! err { ($log:expr, $index:expr, $($args:tt)+) => { diff --git a/ddm/src/mcast.rs b/ddm/src/mcast.rs new file mode 100644 index 000000000..b4691570e --- /dev/null +++ b/ddm/src/mcast.rs @@ -0,0 +1,1248 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Underlay multicast membership programming. +//! +//! Programs the local switch's underlay multicast replication members in the +//! Dendrite (DPD) data plane from the multicast routes DDM has imported from +//! peers. This is the multicast analog of the unicast import-to-DPD path that +//! [`crate::sys::add_underlay_routes`] performs in-process: `ddmd` owns both +//! inputs locally, the imported multicast set ([`Db::imported_mcast`]) and the +//! peer table, so it programs DPD without any cross-daemon coordination. +//! +//! Membership is reconciled by a single periodic sweep over every active +//! underlay group, like the unicast lower-half's resync and `rdb`'s reaper. The +//! control plane never writes multicast members to DPD itself. +//! +//! When a peer's subscription to a group changes, the exchange update handler +//! and peer expiry send the group's address down a notify channel to wake the +//! sweep early. Peer-link resolution does the same for any import that raced +//! ahead of the link. Absent a trigger, the sweep self-ticks on +//! `RECONCILE_INTERVAL`, which bounds how long drift persists. The address +//! on a trigger is only a wake hint: the sweep always reconciles the full set, +//! so a coalesced or missed trigger costs at most one interval of latency. +//! +//! Each sweep recomputes the full, desired member set for every tracked group, +//! repairs any drift, and programs members it could not previously (because the +//! group did not yet exist or a peer link had not resolved). A group whose +//! imports are withdrawn stays in the sweep until its DPD member list is +//! confirmed empty, then drops out, so a withdrawn group is emptied exactly +//! once and the tracked set stays bounded to active and recently active groups. +//! Discovering pre-existing DPD-only groups is deferred for a startup grace so +//! a restart does not empty groups whose imports have not yet been re-learned. +//! +//! DPD's only member-write surface is a full-list replace, so every member edit +//! is a read-modify-write. Groups reconcile concurrently within a pass, but the +//! sweep is the sole writer and each group is a distinct DPD object, so +//! concurrent edits cannot clobber one another. +//! +//! Each imported [`MulticastRoute`] names the peer (`nexthop`) that advertised +//! a group subscription and that peer is a replication target on this sled. +//! Every next hop is resolved to its switch `(PortId, LinkId)` through the +//! interface the peer was discovered on, and members are aggregated per +//! underlay group. +//! +//! Aggregation keys solely on the underlay group address and discards the +//! overlay group. This is sound because Omicron maps each overlay group to a +//! distinct underlay group, so the mapping is one to one and the underlay +//! address alone identifies a group's replication set. A route's overlay group +//! is carried for diagnostics, not for keying. +//! +//! Omicron owns each underlay group's create and delete. `ddmd` programs only +//! the member set of groups that already exist and authorizes each write +//! against the group's tag, read back from DPD, so it never changes the tag or +//! deletes the group. +//! +//! See [RFD 488] for the multicast architecture. +//! +//! [RFD 488]: https://rfd.shared.oxide.computer/rfd/0488 + +use crate::db::Db; +use crate::sm::{DpdConfig, SmContext}; +use crate::sys::DDM_DPD_TAG; +use ddm_api_types::db::MulticastRoute; +use dpd_client::types::{ + Direction, LinkId, MulticastGroupMember, MulticastGroupUpdateUnderlayEntry, + MulticastTag, PortId, UnderlayMulticastIpv6, +}; +use dpd_client::{Client, ClientState}; +use futures::TryStreamExt; +use futures::stream::{self, StreamExt}; +use mg_common::lock; +use reqwest::StatusCode; +use slog::{Logger, debug, error, info, warn}; +use std::collections::{HashMap, HashSet}; +use std::net::Ipv6Addr; +use std::time::{Duration, Instant}; +use tokio::sync::mpsc::{Receiver, Sender}; + +/// Interval between the sweep's periodic membership reconcile passes. +/// +/// A trigger wakes the sweep immediately, so this interval governs only the +/// periodic resync: how quickly drift and any change not delivered as a +/// trigger converge into DPD. It is kept coarse to bound idle DPD churn, since +/// membership changes themselves arrive as triggers. +pub(crate) const RECONCILE_INTERVAL: Duration = Duration::from_secs(10); + +/// Per-request timeout for a single DPD member operation. +/// +/// Bounds one GET or PUT against an unresponsive DPD so a stalled request +/// cannot delay the rest of a sweep pass indefinitely. It is set above the +/// expected DPD member operation latency so that it fires only on a genuine +/// stall, and low enough that a group's sequential fetch-then-write pair +/// (`2 * DPD_REQUEST_TIMEOUT`) stays under [`RECONCILE_INTERVAL`], so a single +/// stalled group cannot extend a pass beyond one reconcile interval. This +/// stall-detection threshold is reasoned about independently of the +/// convergence cadence set by [`RECONCILE_INTERVAL`]. A timed-out operation is +/// logged distinctly and retried on the next pass. +const DPD_REQUEST_TIMEOUT: Duration = Duration::from_secs(3); + +/// Sets how long after startup the sweep waits before discovering DPD-only +/// groups. +/// +/// A restart races the sweep against the exchange machinery: the first pass +/// runs before peers have re-advertised their subscriptions. Seeding the +/// tracked set from DPD immediately would make those groups look withdrawn and +/// drain them, cutting live replication until the imports return. Two +/// reconcile intervals cover peer discovery and the initial pulls. Imported +/// groups are still reconciled immediately, and a group withdrawn after +/// startup drains normally because it is already tracked. +const STARTUP_SEED_GRACE: Duration = RECONCILE_INTERVAL.saturating_mul(2); + +/// Cap on concurrently reconciling groups within a single pass. +/// +/// Bounds the in-flight DPD requests a pass can generate so a large tracked +/// set cannot flood DPD with an unbounded burst of GETs and PUTs. +const MAX_CONCURRENT_GROUP_RECONCILES: usize = 16; + +/// Run the multicast membership sweep. +/// +/// This loops forever, so callers spawn it as a dedicated task on the runtime. +/// +/// We track the set of active underlay groups and reconcile them on each pass. +/// `notify_rx` is a wake hint only. A trigger wakes the sweep early, and absent +/// a trigger it self-ticks on `RECONCILE_INTERVAL`. Every pass reconciles the +/// full tracked set, so the group address carried by a trigger is not consulted +/// and a coalesced trigger costs at most one interval of latency. +/// +/// The tracked set is the union of every currently imported group and any group +/// still being drained. `reconcile_group` returns `false` only once a +/// withdrawn group's DPD members are confirmed empty, so that a group leaves +/// the set exactly once its drain is complete. A re-import re-adds it on the +/// next pass since the control plane writes to the DB before sending its +/// trigger. +/// +/// `peers` is the set of per-interface state machine contexts, fixed at +/// startup. Peer identity lives behind interior mutability, so each pass +/// resolves against whatever peers have been discovered when it reads. +/// +/// Under `--api-only` there are no state machines, so the set is empty. +pub async fn run( + db: Db, + peers: Vec, + dpd: DpdConfig, + mut notify_rx: Receiver, + log: Logger, +) { + let client_state = ClientState { + tag: DDM_DPD_TAG.into(), + log: log.clone(), + }; + // Build the inner HTTP client explicitly to bound each request at + // DPD_REQUEST_TIMEOUT. The progenitor-generated dpd_client defaults to a + // 15s connect and request timeout, which exceeds RECONCILE_INTERVAL. A + // single stalled GET could outlast the whole reconcile interval, and a + // sequential fetch-then-write pair could run three times it. Stepping down + // to DPD_REQUEST_TIMEOUT keeps a stalled group's pair under one interval. + let http = match reqwest::ClientBuilder::new() + .connect_timeout(DPD_REQUEST_TIMEOUT) + .timeout(DPD_REQUEST_TIMEOUT) + .build() + { + Ok(http) => http, + Err(e) => { + error!(log, "failed to build DPD HTTP client, stopping sweep: {e}"); + return; + } + }; + let client = Client::new_with_client( + &format!("http://{}:{}", dpd.host, dpd.port), + http, + client_state, + ); + + // On each pass, the sweep reconciles every imported group plus any group + // still draining withdrawn members, so the set stays bounded to active and + // recently active groups rather than growing without limit. We seed it from + // the underlay groups DPD already has members for. + // + // On a fresh start, the imported set and triggers only reference groups + // with live subscriptions, so a group whose imports were withdrawn while + // `ddmd` was down would never re-enter the sweep and its stale replication + // members would persist. After a startup grace, folding those groups in + // lets a pass drain any that no peer still imports, while groups still + // imported simply reconcile as usual. A failed listing is retried after + // another reconcile interval, since ordinary passes cannot discover + // orphans. + let mut tracked: HashSet = HashSet::new(); + let mut seeded = false; + let mut next_seed_attempt = Instant::now() + STARTUP_SEED_GRACE; + + loop { + if !seeded && Instant::now() >= next_seed_attempt { + match client.member_group_ips(&log).await { + Some(groups) => { + tracked.extend(groups); + seeded = true; + } + None => { + next_seed_attempt = Instant::now() + RECONCILE_INTERVAL; + } + } + } + + // The imported set and resolved peer links are the same for every group + // in a pass, so compute them once here rather than per group. + let imported = db.imported_mcast(); + let peer_links = resolve_peer_links(&peers, &log); + + tracked = + reconcile_pass(tracked, imported, peer_links, &client, &log).await; + + // Wait for a trigger or one idle reconcile interval, whichever comes + // first, then drain any burst since the next pass reconciles + // everything. A fresh sleep is sufficient because every trigger also + // runs a full pass, and avoids catch-up timer semantics after a slow + // pass. Both arms are cancel-safe leaf futures with no `.await` in + // their bodies, so the sweep cannot futurelock. + tokio::select! { + trigger = notify_rx.recv() => match trigger { + Some(_) => while notify_rx.try_recv().is_ok() {}, + None => { + // Unreachable while `ddmd` runs: `main()` owns the original + // `notify_tx` and parks for the daemon's lifetime, so the + // channel cannot close even if every per-peer sender clone + // is torn down. We stop the sweep rather than spin on a + // closed channel if that invariant ever changes. + error!(log, "multicast notify channel closed, stopping sweep"); + break; + } + }, + _ = tokio::time::sleep(RECONCILE_INTERVAL) => {} + } + } +} + +/// Signal the multicast sweep for each distinct underlay group in `routes`. +/// +/// The route iterator may repeat a group for multiple next hops, so group +/// addresses are deduplicated before notification. +pub(crate) fn notify_affected_groups<'a>( + routes: impl IntoIterator, + notify: &Sender, +) { + let groups = routes + .into_iter() + .map(|route| route.origin.underlay_group.ip()) + .collect(); + notify_groups(groups, notify); +} + +/// Wake the multicast sweep once per group in `groups`. +fn notify_groups(groups: HashSet, notify: &Sender) { + for group in groups { + // A full channel or closed receiver is harmless because triggers are + // wake hints and the periodic pass reconciles the full tracked set. + let _ = notify.try_send(group); + } +} + +/// Wake the multicast sweep for every group `peer` advertised once that peer's +/// link resolves. +/// +/// A multicast import already wakes the sweep, but a route imported before the +/// peer link resolved cannot be programmed yet, so it waits out the reconcile +/// interval. Waking the peer's groups on resolution closes that window. The +/// imported set is read, not consumed, so this is the non-destructive analog of +/// the [`Db::remove_nexthop_routes`] removal on peer expiry. +pub(crate) fn notify_peer_groups( + db: &Db, + peer: Ipv6Addr, + notify: &Sender, +) { + notify_groups(db.mcast_groups_for_nexthop(peer), notify); +} + +/// Reconcile every tracked group against DPD once, returning the next tracked +/// set. +/// +/// Folds every currently imported group into `tracked`, reconciles the whole +/// set concurrently, and returns only the groups `reconcile_group` reports as +/// still active. A withdrawn group lingers for exactly one pass to empty its +/// DPD members, then drops out on the following pass. Re-importing a dropped +/// group re-adds it here, since Omicron writes to the DB before triggering the +/// sweep. +/// +/// The per-group futures run concurrently on this task rather than being +/// spawned, capped at [`MAX_CONCURRENT_GROUP_RECONCILES`] in flight, so a +/// group whose DPD call stalls does not serialize the others behind it and a +/// large tracked set cannot flood DPD. +/// +/// The pass still returns only once its slowest group completes. Each request +/// is bounded by [`DPD_REQUEST_TIMEOUT`], but stalled groups beyond the +/// concurrency cap execute in waves, so a pass is bounded by one timeout per +/// wave of stalled groups rather than one timeout overall. +async fn reconcile_pass( + mut tracked: HashSet, + imported: HashSet, + peer_links: HashMap, + client: &C, + log: &Logger, +) -> HashSet { + for route in imported.iter() { + tracked.insert(route.origin.underlay_group.ip()); + } + + // Borrow once so every per-group future shares the same imports and + // resolved links by reference. + let imported = &imported; + let peer_links = &peer_links; + stream::iter(tracked) + .map(|group_ip| async move { + ( + group_ip, + reconcile_group(group_ip, imported, peer_links, client, log) + .await, + ) + }) + .buffer_unordered(MAX_CONCURRENT_GROUP_RECONCILES) + .filter_map(|(group_ip, keep)| async move { keep.then_some(group_ip) }) + .collect() + .await +} + +/// Whether a DPD client error is a request timeout. +/// +/// A timeout surfaces as a transport-level error with no HTTP status, so it is +/// distinguished by inspecting the underlying `reqwest::Error` rather than by +/// status code. +fn is_timeout(e: &dpd_client::Error) -> bool { + matches!(e, dpd_client::Error::CommunicationError(re) if re.is_timeout()) +} + +/// Outcome of writing a group's member list to DPD. +#[derive(Clone, Copy)] +enum WriteOutcome { + /// Members were written. + Updated, + /// The group disappeared or its tag changed after the preceding read. + Stale, + /// The write failed and should be retried on the next pass. + Retry, +} + +/// Outcome of reading a group's state from DPD. +#[derive(Clone)] +enum FetchOutcome { + /// The group exists and its authorization tag and current members were read. + Found(String, Vec), + /// The group does not exist in DPD, either because Omicron has not created + /// it yet or because it has been deleted. + Absent, + /// The read failed, so the group's state is unknown this pass. + Retry, +} + +/// DPD group operations the reconcile loop depends on. +trait GroupClient { + /// Read an underlay group's current members and authorization tag. + async fn fetch_group( + &self, + log: &Logger, + group_ip: Ipv6Addr, + ) -> FetchOutcome; + + /// Write `members` to an underlay group, authorized by its current `tag`. + async fn write_members( + &self, + log: &Logger, + group_ip: Ipv6Addr, + tag: &str, + members: Vec, + ) -> WriteOutcome; + + /// Underlay groups that currently have members programmed in DPD, or + /// `None` if the listing failed. + /// + /// Read after the startup grace to seed the sweep's tracked set. `ddmd` is + /// the sole writer of underlay members on this switch, so every group + /// returned was programmed by `ddmd`, possibly before a restart, and is + /// safe to fold in. A failed listing is retried later, since ordinary + /// passes reconcile only tracked and imported groups and cannot otherwise + /// discover orphans. + async fn member_group_ips(&self, log: &Logger) -> Option>; +} + +impl GroupClient for Client { + /// Distinguishes a group that is genuinely absent (`FetchOutcome::Absent`) + /// from one whose state could not be read (`FetchOutcome::Retry`), so a + /// withdrawn group is not dropped from the sweep on a transient read failure + /// before its members are confirmed drained. + async fn fetch_group( + &self, + log: &Logger, + group_ip: Ipv6Addr, + ) -> FetchOutcome { + let underlay_ip = UnderlayMulticastIpv6::from(group_ip); + match self.multicast_group_get_underlay(&underlay_ip).await { + Ok(resp) => { + let resp = resp.into_inner(); + FetchOutcome::Found(resp.tag, resp.members) + } + // The underlay group's create and delete are owned by Omicron, which + // creates the group before traffic flows. Until the group exists + // there are no members to program, so skip it. + Err(e) if e.status() == Some(StatusCode::NOT_FOUND) => { + debug!( + log, + "underlay group {group_ip} does not exist yet, skipping \ + until Omicron creates it" + ); + FetchOutcome::Absent + } + + // Surface a stalled read distinctly from other failures. The sweep + // retries the group on its next pass regardless. + Err(e) if is_timeout(&e) => { + warn!( + log, + "get of underlay group {group_ip} timed out after \ + {DPD_REQUEST_TIMEOUT:?}, retrying next pass" + ); + FetchOutcome::Retry + } + Err(e) => { + warn!(log, "failed to get underlay group {group_ip}: {e}"); + FetchOutcome::Retry + } + } + } + + /// The expected races, a tag change (403) or a deleted group (404), are + /// returned as outcomes rather than logged, leaving the handling to the + /// caller. + async fn write_members( + &self, + log: &Logger, + group_ip: Ipv6Addr, + tag: &str, + members: Vec, + ) -> WriteOutcome { + let underlay_ip = UnderlayMulticastIpv6::from(group_ip); + let tag = match MulticastTag::try_from(tag.to_string()) { + Ok(tag) => tag, + Err(e) => { + error!( + log, + "tag for underlay group {group_ip} is invalid, skipping \ + update: {e}" + ); + return WriteOutcome::Retry; + } + }; + + let body = MulticastGroupUpdateUnderlayEntry { members }; + match self + .multicast_group_update_underlay(&underlay_ip, &tag, &body) + .await + { + Ok(_) => WriteOutcome::Updated, + Err(e) + if matches!( + e.status(), + Some(StatusCode::FORBIDDEN | StatusCode::NOT_FOUND) + ) => + { + WriteOutcome::Stale + } + // Log a stalled write distinctly from other failures, while both + // share the same retry outcome. + Err(e) if is_timeout(&e) => { + warn!( + log, + "update of underlay group {group_ip} members timed out \ + after {DPD_REQUEST_TIMEOUT:?}, retrying next pass" + ); + WriteOutcome::Retry + } + Err(e) => { + warn!( + log, + "failed to update underlay group {group_ip} members: {e}" + ); + WriteOutcome::Retry + } + } + } + + async fn member_group_ips(&self, log: &Logger) -> Option> { + let groups: Vec = match self + .multicast_groups_list_stream(None) + .try_collect() + .await + { + Ok(groups) => groups, + Err(e) => { + warn!( + log, + "could not list multicast groups to seed sweep, retrying \ + later: {e}" + ); + return None; + } + }; + + Some( + groups + .into_iter() + .filter_map(|group| match group { + dpd_client::types::MulticastGroupResponse::Underlay { + group_ip, + members, + .. + } if !members.is_empty() => Some(*group_ip), + _ => None, + }) + .collect(), + ) + } +} + +/// Resolve each established peer's underlay address to a switch +/// `(PortId, LinkId)` through the interface it was discovered on. +/// +/// Peers without an established identity, without an interface name, or whose +/// interface does not resolve to a switch link are omitted. +/// +/// A peer omitted here is seen by `group_members` as an unresolved next hop, so +/// a transient resolution failure neither drops a previously programmed member +/// nor blocks a newly resolved one. +/// +/// The map keys on the peer's link-local address alone, with no interface +/// scope, since an imported route's `nexthop` carries no interface either. +/// This relies on the rack deriving link-local addresses from EUI-64, which +/// makes them unique across links rather than only within one ([RFC 4007], +/// section 5). A duplicate address on distinct links would collapse to one +/// entry, so a collision is logged as a warning. +/// +/// [RFC 4007]: https://www.rfc-editor.org/rfc/rfc4007#section-5 +fn resolve_peer_links( + peers: &[SmContext], + log: &Logger, +) -> HashMap { + let mut peer_links: HashMap = HashMap::new(); + for sm in peers { + let Some(peer) = lock!(sm.iface.peer_identity).clone() else { + continue; + }; + let if_name = lock!(sm.iface.if_name).clone(); + if if_name.is_empty() { + warn!( + log, + "peer {} has no interface name; omitting as multicast member", + peer.addr + ); + continue; + } + + match mg_common::tfport::port_link_from_ifname(&if_name) { + Ok(port_link) => { + if let Some(prev) = + peer_links.insert(peer.addr, port_link.clone()) + && prev != port_link + { + warn!( + log, + "peer link-local address {} resolves to multiple \ + switch links ({prev:?} and {port_link:?}), violating \ + EUI-64 uniqueness; keeping the latter", + peer.addr + ); + } + } + Err(e) => warn!( + log, + "cannot resolve peer {} interface {if_name} to a switch link, \ + omitting as multicast member: {e}", + peer.addr + ), + } + } + peer_links +} + +/// Aggregate one underlay group's desired replication members. +/// +/// Returns the member list for `group_ip` and whether any of its next hops +/// failed to resolve this pass, in which case the derived set may be incomplete. +/// +/// Members are derived from each route's `nexthop`. Distinct downstream peers +/// carry distinct next hops, so each becomes its own member. Subscribers reached +/// through the same downstream peer collapse to one member, because a single +/// egress port toward that peer suffices and the next hop handles further +/// fan-out. The path vector is not needed, only the per-node egress set. +fn group_members( + group_ip: Ipv6Addr, + imported: &HashSet, + peer_links: &HashMap, +) -> (Vec, bool) { + let mut members: Vec = Vec::new(); + let mut has_unresolved = false; + for route in imported + .iter() + .filter(|route| route.origin.underlay_group.ip() == group_ip) + { + let Some((port_id, link_id)) = peer_links.get(&route.nexthop) else { + has_unresolved = true; + continue; + }; + // The single (port, link) here is per next hop, not per group: one peer + // is reached over one tfport link. A group fans out to as many links as + // it has distinct downstream peers. + let member = MulticastGroupMember { + port_id: port_id.clone(), + link_id: *link_id, + direction: Direction::Underlay, + }; + if !members.contains(&member) { + members.push(member); + } + } + (members, has_unresolved) +} + +/// Reconcile a single underlay group's members in DPD against the multicast +/// routes DDM has imported, returning whether the group is still active. +/// +/// The group's current members are read fresh from DPD and diffed against the +/// desired set, so the periodic resync repairs member drift. +/// +/// Returns `true` to keep the group tracked, either while it still has imports +/// (so resync repairs drift) or whenever its DPD state could not be read this +/// pass, and `false` only once the group has no imports and its DPD member list +/// is confirmed empty, so it drops out of the sweep. +async fn reconcile_group( + group_ip: Ipv6Addr, + imported: &HashSet, + peer_links: &HashMap, + client: &C, + log: &Logger, +) -> bool { + let has_imports = imported + .iter() + .any(|route| route.origin.underlay_group.ip() == group_ip); + let (members, has_unresolved) = + group_members(group_ip, imported, peer_links); + + let (tag, existing) = match client.fetch_group(log, group_ip).await { + FetchOutcome::Found(tag, existing) => (tag, existing), + // The group is absent from DPD, Omicron has not created it or it is + // deleted. There is nothing to program or drain, so keep it tracked + // only while it still has imports, so a later pass programs it once it + // exists. + FetchOutcome::Absent => return has_imports, + // The state is unknown at this pass. Keep the group tracked so the next + // pass retries, whether it is active or still draining withdrawn + // members. A withdrawn group must not drop out here, or stale + // replication would stay programmed until some later re-import tracked + // it again. + FetchOutcome::Retry => return true, + }; + + if !has_imports { + // Withdrawn: empty the member list to stop replication, leaving the + // group for Omicron to delete. + if existing.is_empty() { + return false; + } + return match client.write_members(log, group_ip, &tag, Vec::new()).await + { + WriteOutcome::Updated => { + info!( + log, + "emptied withdrawn underlay group {group_ip} members" + ); + false + } + // Already gone, or recreated under a tag that is no longer what + // we've seen. Either way `ddmd` no longer programs this group. + WriteOutcome::Stale => false, + // Retry the empty on the next pass. + WriteOutcome::Retry => true, + }; + } + + // When a next hop did not resolve this pass, the derived set may be missing + // members. Merge it with the group's current members so a transient + // resolution failure neither drops a previously programmed member nor + // blocks adding a newly resolved one. With every next hop resolved, the + // derived set replaces the current members. + // + // Fail open: preserve every current member, not only those of unresolved + // routes, since a member cannot be attributed to a next hop without + // resolving it. A stale member, even one unrelated to the unresolved next + // hop, persists until every next hop resolves. Extra replication is + // preferred over dropping a live member. + let to_write = if has_unresolved { + let merged = union_members(&members, &existing); + if merged.len() > members.len() { + debug!( + log, + "underlay group {group_ip} has unresolved next hops, preserving \ + {} current DPD member(s) beyond the {} derived this pass", + merged.len() - members.len(), + members.len() + ); + } + merged + } else { + members + }; + + if !members_eq(&existing, &to_write) { + match client.write_members(log, group_ip, &tag, to_write).await { + WriteOutcome::Updated => { + info!(log, "updated underlay group {group_ip} members") + } + WriteOutcome::Stale | WriteOutcome::Retry => {} + } + } + + true +} + +/// Union of two multicast member lists, preserving order and dropping +/// duplicates. Used to merge a derived member set with the group's current DPD +/// members when a next hop did not resolve this pass. +fn union_members( + base: &[MulticastGroupMember], + extra: &[MulticastGroupMember], +) -> Vec { + let mut merged = base.to_vec(); + for member in extra { + if !merged.contains(member) { + merged.push(member.clone()); + } + } + merged +} + +/// Compare two multicast member lists for set equality, ignoring order and +/// duplicates. +/// +/// `ddmd` never writes duplicates, but the list read back from DPD is not +/// trusted to be duplicate-free. A length check alone would be fooled by +/// duplicates, e.g. `[A, A]` against `[A, B]`, skipping the write that would +/// repair the drift, so containment is checked in both directions. +fn members_eq(a: &[MulticastGroupMember], b: &[MulticastGroupMember]) -> bool { + a.iter().all(|member| b.contains(member)) + && b.iter().all(|member| a.contains(member)) +} + +#[cfg(test)] +mod tests { + use super::*; + use ddm_api_types::net::{ + MulticastOrigin, OverlayMulticast, UnderlayMulticastIpv6, Vni, + }; + use std::net::IpAddr; + + fn underlay(last: u16) -> Ipv6Addr { + Ipv6Addr::new(0xff04, 0, 0, 0, 0, 0, 0, last) + } + + fn route(nexthop: Ipv6Addr, group: Ipv6Addr) -> MulticastRoute { + MulticastRoute { + origin: MulticastOrigin { + overlay_group: OverlayMulticast::new(IpAddr::V6( + Ipv6Addr::new(0xff0e, 0, 0, 0, 0, 0, 0, 1), + )) + .unwrap(), + underlay_group: UnderlayMulticastIpv6::new(group).unwrap(), + vni: Vni::DEFAULT_MULTICAST, + metric: 0, + source: None, + }, + nexthop, + path: Vec::new(), + } + } + + fn rear(port: &str, link: u8) -> (PortId, LinkId) { + ( + PortId::Rear(dpd_client::types::Rear::try_from(port).unwrap()), + LinkId(link), + ) + } + + fn member(port: &str, link: u8) -> MulticastGroupMember { + let (port_id, link_id) = rear(port, link); + MulticastGroupMember { + port_id, + link_id, + direction: Direction::Underlay, + } + } + + #[test] + fn distinct_peers_become_distinct_members() { + let peer_a = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let peer_b = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 2); + let group = underlay(1); + + let imported = + HashSet::from([route(peer_a, group), route(peer_b, group)]); + let peer_links = HashMap::from([ + (peer_a, rear("rear0", 0)), + (peer_b, rear("rear1", 0)), + ]); + + let (members, unresolved) = + group_members(group, &imported, &peer_links); + assert!(!unresolved); + assert_eq!(members.len(), 2); + assert!(members.contains(&member("rear0", 0))); + assert!(members.contains(&member("rear1", 0))); + } + + #[test] + fn same_link_peers_share_member() { + let peer_a = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let peer_b = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 2); + let group = underlay(1); + + // Two distinct peers that resolve to the same switch link. Replicating + // twice out one (PortId, LinkId) would duplicate delivery on that link, + // so the members collapse to one. The dedup keys on the resolved link, + // not on the next hop. + let imported = + HashSet::from([route(peer_a, group), route(peer_b, group)]); + let peer_links = HashMap::from([ + (peer_a, rear("rear0", 0)), + (peer_b, rear("rear0", 0)), + ]); + + let (members, unresolved) = + group_members(group, &imported, &peer_links); + assert!(!unresolved); + assert_eq!(members, vec![member("rear0", 0)]); + } + + #[test] + fn unresolved_nexthop_returns_empty_and_unresolved() { + let peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let group = underlay(7); + + let imported = HashSet::from([route(peer, group)]); + // No peer_links entry: next hop is unresolved. + let peer_links = HashMap::new(); + + let (members, unresolved) = + group_members(group, &imported, &peer_links); + assert!(members.is_empty()); + assert!(unresolved); + } + + #[test] + fn mixed_resolution_returns_members_and_unresolved() { + let resolved = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let unresolved_peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 2); + let group = underlay(3); + + // One next hop resolves and one does not. The resolved peer contributes a + // member, and the group is still flagged so the reconcile merges with the + // group's current DPD members rather than dropping the unresolved one. + let imported = HashSet::from([ + route(resolved, group), + route(unresolved_peer, group), + ]); + let peer_links = HashMap::from([(resolved, rear("rear0", 0))]); + + let (members, unresolved) = + group_members(group, &imported, &peer_links); + assert!(unresolved); + assert_eq!(members, vec![member("rear0", 0)]); + } + + /// Mock DPD that returns preset fetch and write outcomes, and records every + /// member list written so a test can assert the reconcile's keep/drop + /// decision and whether it wrote at all. + /// + /// A landed write (`WriteOutcome::Updated`) updates the stored fetch state + /// to the members just written, so a later fetch reflects it. This models + /// DPD's read-after-write semantics and lets a multi-pass test observe a + /// group's drain across passes. + struct MockDpd { + fetch: std::sync::Mutex, + write_outcome: WriteOutcome, + writes: std::sync::Mutex>>, + member_groups: Vec, + } + + impl MockDpd { + fn new(fetch: FetchOutcome, write_outcome: WriteOutcome) -> Self { + Self { + fetch: std::sync::Mutex::new(fetch), + write_outcome, + writes: std::sync::Mutex::new(Vec::new()), + member_groups: Vec::new(), + } + } + + fn with_member_groups(mut self, groups: Vec) -> Self { + self.member_groups = groups; + self + } + + fn writes(&self) -> Vec> { + self.writes.lock().unwrap().clone() + } + } + + impl GroupClient for MockDpd { + async fn fetch_group( + &self, + _log: &Logger, + _group_ip: Ipv6Addr, + ) -> FetchOutcome { + self.fetch.lock().unwrap().clone() + } + + async fn write_members( + &self, + _log: &Logger, + _group_ip: Ipv6Addr, + tag: &str, + members: Vec, + ) -> WriteOutcome { + self.writes.lock().unwrap().push(members.clone()); + if matches!(self.write_outcome, WriteOutcome::Updated) { + *self.fetch.lock().unwrap() = + FetchOutcome::Found(tag.to_string(), members); + } + self.write_outcome + } + + async fn member_group_ips( + &self, + _log: &Logger, + ) -> Option> { + Some(self.member_groups.clone()) + } + } + + fn found(members: Vec) -> FetchOutcome { + FetchOutcome::Found("tag".to_string(), members) + } + + fn reconcile( + group: Ipv6Addr, + imported: &HashSet, + peer_links: &HashMap, + mock: &MockDpd, + ) -> bool { + let log = Logger::root(slog::Discard, slog::o!()); + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + rt.block_on(reconcile_group(group, imported, peer_links, mock, &log)) + } + + fn run_pass( + tracked: HashSet, + imported: &HashSet, + peer_links: &HashMap, + mock: &MockDpd, + ) -> HashSet { + let log = Logger::root(slog::Discard, slog::o!()); + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + rt.block_on(reconcile_pass( + tracked, + imported.clone(), + peer_links.clone(), + mock, + &log, + )) + } + + /// Drives the sweep's cross-pass carry-over invariant: an active group is + /// tracked, a withdraw takes one pass to empty its members and drop it, and + /// a re-import adds and reconciles it again. + #[test] + fn group_drains_and_readds_across_passes() { + let peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let group = underlay(1); + let active = HashSet::from([route(peer, group)]); + let withdrawn = HashSet::new(); + let peer_links = HashMap::from([(peer, rear("rear0", 0))]); + + // DPD already holds the derived member, so the group starts active and + // in sync with the imported set. + let mock = MockDpd::new( + found(vec![member("rear0", 0)]), + WriteOutcome::Updated, + ); + + // Pass 1: imported and already in sync, so the group is tracked without + // a write. + let tracked = run_pass(HashSet::new(), &active, &peer_links, &mock); + assert_eq!(tracked, HashSet::from([group])); + assert!(mock.writes().is_empty()); + + // Pass 2: the withdrawn group carries over from pass 1, its members are + // emptied, and then it drops out of the tracked set. + let tracked = run_pass(tracked, &withdrawn, &peer_links, &mock); + assert!(tracked.is_empty()); + assert_eq!(mock.writes(), vec![Vec::::new()]); + + // Pass 3: the dropped group is re-imported and reprogrammed, since DPD + // now holds no members for it. + let tracked = run_pass(tracked, &active, &peer_links, &mock); + assert_eq!(tracked, HashSet::from([group])); + assert_eq!(mock.writes(), vec![Vec::new(), vec![member("rear0", 0)]]); + } + + /// A group whose imports were withdrawn while `ddmd` was down has no entry + /// in the imported set or any trigger, so only the startup seed can + /// re-initialize it. + #[test] + fn startup_seed_drains_orphans() { + let group = underlay(9); + let mock = MockDpd::new( + found(vec![member("rear0", 0)]), + WriteOutcome::Updated, + ) + .with_member_groups(vec![group]); + let log = Logger::root(slog::Discard, slog::o!()); + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + let seeded: HashSet = rt + .block_on(mock.member_group_ips(&log)) + .unwrap() + .into_iter() + .collect(); + assert!(seeded.contains(&group)); + + let next = run_pass(seeded, &HashSet::new(), &HashMap::new(), &mock); + assert_eq!(mock.writes(), vec![Vec::::new()]); + assert!(next.is_empty()); + } + + #[test] + fn absent_group_with_imports_stays_tracked() { + let peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let group = underlay(1); + let imported = HashSet::from([route(peer, group)]); + let peer_links = HashMap::from([(peer, rear("rear0", 0))]); + let mock = MockDpd::new(FetchOutcome::Absent, WriteOutcome::Updated); + + // Omicron has not created the group yet, so there is nothing to program, + // but it stays tracked so a later pass programs it once it exists. + assert!(reconcile(group, &imported, &peer_links, &mock)); + assert!(mock.writes().is_empty()); + } + + #[test] + fn absent_group_without_imports_drops() { + let group = underlay(1); + let imported = HashSet::new(); + let peer_links = HashMap::new(); + let mock = MockDpd::new(FetchOutcome::Absent, WriteOutcome::Updated); + + assert!(!reconcile(group, &imported, &peer_links, &mock)); + assert!(mock.writes().is_empty()); + } + + #[test] + fn read_retry_keeps_withdrawn_group() { + let group = underlay(1); + let imported = HashSet::new(); + let peer_links = HashMap::new(); + let mock = MockDpd::new(FetchOutcome::Retry, WriteOutcome::Updated); + + // A withdrawn group must not drop out on a transient read failure, or + // its stale replication would stay programmed until a later re-import. + assert!(reconcile(group, &imported, &peer_links, &mock)); + assert!(mock.writes().is_empty()); + } + + #[test] + fn withdrawn_group_with_no_members_drops_without_writing() { + let group = underlay(1); + let imported = HashSet::new(); + let peer_links = HashMap::new(); + let mock = MockDpd::new(found(Vec::new()), WriteOutcome::Updated); + + assert!(!reconcile(group, &imported, &peer_links, &mock)); + assert!(mock.writes().is_empty()); + } + + #[test] + fn withdrawn_group_drains_then_drops() { + let group = underlay(1); + let imported = HashSet::new(); + let peer_links = HashMap::new(); + let mock = MockDpd::new( + found(vec![member("rear0", 0)]), + WriteOutcome::Updated, + ); + + assert!(!reconcile(group, &imported, &peer_links, &mock)); + assert_eq!(mock.writes(), vec![Vec::::new()]); + } + + #[test] + fn empty_write_retry_keeps_withdrawn_group() { + let group = underlay(1); + let imported = HashSet::new(); + let peer_links = HashMap::new(); + let mock = + MockDpd::new(found(vec![member("rear0", 0)]), WriteOutcome::Retry); + + assert!(reconcile(group, &imported, &peer_links, &mock)); + assert_eq!(mock.writes(), vec![Vec::::new()]); + } + + #[test] + fn stale_empty_write_drops_withdrawn_group() { + let group = underlay(1); + let imported = HashSet::new(); + let peer_links = HashMap::new(); + let mock = + MockDpd::new(found(vec![member("rear0", 0)]), WriteOutcome::Stale); + + // The group disappeared or was reassigned after the read, so `ddmd` + // abandons the withdrawn group rather than retrying. + assert!(!reconcile(group, &imported, &peer_links, &mock)); + assert_eq!(mock.writes(), vec![Vec::::new()]); + } + + #[test] + fn active_group_with_matching_members_skips_write() { + let peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let group = underlay(1); + let imported = HashSet::from([route(peer, group)]); + let peer_links = HashMap::from([(peer, rear("rear0", 0))]); + let mock = MockDpd::new( + found(vec![member("rear0", 0)]), + WriteOutcome::Updated, + ); + + assert!(reconcile(group, &imported, &peer_links, &mock)); + assert!(mock.writes().is_empty()); + } + + #[test] + fn active_group_with_drifted_members_writes_derived_set() { + let peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let group = underlay(1); + let imported = HashSet::from([route(peer, group)]); + let peer_links = HashMap::from([(peer, rear("rear0", 0))]); + let mock = MockDpd::new(found(Vec::new()), WriteOutcome::Updated); + + assert!(reconcile(group, &imported, &peer_links, &mock)); + assert_eq!(mock.writes(), vec![vec![member("rear0", 0)]]); + } + + #[test] + fn unresolved_active_group_preserves_members() { + let peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let group = underlay(1); + let imported = HashSet::from([route(peer, group)]); + // No peer_links entry: the next hop is unresolved this pass. + let peer_links = HashMap::new(); + let mock = MockDpd::new( + found(vec![member("rear0", 0)]), + WriteOutcome::Updated, + ); + + // The derived set is empty because the next hop did not resolve, but + // the merge with current DPD members keeps the programmed member, so no + // destructive write occurs. + assert!(reconcile(group, &imported, &peer_links, &mock)); + assert!(mock.writes().is_empty()); + } + + /// Encodes the fail-open merge policy: while any next hop is unresolved, + /// a stale DPD member that no import accounts for, even one unrelated to + /// the unresolved next hop, persists rather than being dropped. + #[test] + fn unresolved_nexthop_retains_unrelated_stale_member() { + let resolved = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let unresolved_peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 2); + let group = underlay(1); + let imported = HashSet::from([ + route(resolved, group), + route(unresolved_peer, group), + ]); + let peer_links = HashMap::from([(resolved, rear("rear0", 0))]); + + // DPD holds the resolved member plus a stale one ("rear1") that no + // current import accounts for. + let mock = MockDpd::new( + found(vec![member("rear0", 0), member("rear1", 0)]), + WriteOutcome::Updated, + ); + + // The stale member cannot be distinguished from one owned by the + // unresolved next hop, so the merge preserves it and no write occurs. + assert!(reconcile(group, &imported, &peer_links, &mock)); + assert!(mock.writes().is_empty()); + } + + #[test] + fn stale_write_keeps_active_group() { + let peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let group = underlay(1); + let imported = HashSet::from([route(peer, group)]); + let peer_links = HashMap::from([(peer, rear("rear0", 0))]); + let mock = MockDpd::new(found(Vec::new()), WriteOutcome::Stale); + + // The group changed after the read, but remains imported, so it stays + // tracked to retry with a fresh read next pass. + assert!(reconcile(group, &imported, &peer_links, &mock)); + assert_eq!(mock.writes(), vec![vec![member("rear0", 0)]]); + } + + #[test] + fn notify_deduplicates_groups() { + let group_a = underlay(1); + let group_b = underlay(2); + let peer_a = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let peer_b = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 2); + + let routes = [ + route(peer_a, group_a), + route(peer_b, group_a), + route(peer_a, group_b), + ]; + + let (tx, mut rx) = + tokio::sync::mpsc::channel(crate::MCAST_NOTIFY_CHANNEL_DEPTH); + notify_affected_groups(routes.iter(), &tx); + + let mut signalled = Vec::new(); + while let Ok(group) = rx.try_recv() { + signalled.push(group); + } + assert_eq!(signalled.len(), 2); + assert_eq!( + signalled.into_iter().collect::>(), + HashSet::from([group_a, group_b]) + ); + } +} diff --git a/ddm/src/sm/mod.rs b/ddm/src/sm/mod.rs index 4a52efb55..160cabd0b 100644 --- a/ddm/src/sm/mod.rs +++ b/ddm/src/sm/mod.rs @@ -4,13 +4,13 @@ //! State machine type definitions and the [`StateMachine`] handle. The //! routing state machine implementation (discovery, solicit, exchange) lives -//! in the [`state`] submodule and is illumos-only, since it programs kernel -//! routes via [`crate::sys`] and reads interface addressing through `libnet`. +//! in the `state` submodule and is illumos-only, since it programs kernel +//! routes via `crate::sys` and reads interface addressing through `libnet`. use crate::db::Db; use crate::discovery::{self, Version}; use ddm_api_types::db::{PeerStatus, RouterKind}; -use ddm_api_types::net::TunnelOrigin; +use ddm_api_types::net::{MulticastOrigin, TunnelOrigin}; use mg_common::lock; use oxnet::Ipv6Net; use slog::Logger; @@ -33,6 +33,17 @@ pub enum AdminEvent { /// Withdraw a set of IPv6 prefixes Withdraw(PrefixSet), + /// Announce a set of multicast origins to peers. + AnnounceMulticast(HashSet), + + /// Withdraw a set of multicast origins. Each state machine revalidates + /// remaining reachability against the database when it processes the + /// event, rather than acting on a snapshot captured at request time. + /// The modification lands before this event is enqueued, so the processing- + /// time read is guaranteed to observe the withdrawal, and it also + /// observes any later import that has since restored reachability. + WithdrawMulticast(HashSet), + /// Expire the peer at the specified address Expire(Ipv6Addr), @@ -48,7 +59,7 @@ pub enum PrefixSet { #[derive(Debug)] pub enum PeerEvent { - Push(ddm_protocol::v3::Update), + Push(Arc), } #[derive(Debug)] @@ -187,6 +198,11 @@ pub struct InterfaceState { pub fsm_state: Mutex, pub last_fsm_state_change: Mutex, pub peer_identity: Mutex>, + /// Orders route application with expiry and renumber cleanup for this + /// interface. Discovery must not take this lock: route programming can + /// perform network and kernel I/O, while discovery needs to keep updating + /// peer liveness independently. + pub route_update: Mutex<()>, } impl InterfaceState { @@ -218,6 +234,7 @@ impl Default for InterfaceState { fsm_state: Mutex::new(FsmState::Init), last_fsm_state_change: Mutex::new(Instant::now()), peer_identity: Mutex::new(None), + route_update: Mutex::new(()), } } } @@ -251,6 +268,11 @@ pub struct SmContext { pub hostname: String, pub iface: Arc, pub stats: Arc, + /// Notifies the `crate::mcast` sweep that an underlay group's imported + /// membership changed, by sending the group's address. The sweep wakes early + /// to reconcile the group's DPD members, so the control plane never touches + /// DPD directly. + pub mcast_notify: tokio::sync::mpsc::Sender, pub log: Logger, } diff --git a/ddm/src/sm/state.rs b/ddm/src/sm/state.rs index ecb8ffa68..8c379c9ed 100644 --- a/ddm/src/sm/state.rs +++ b/ddm/src/sm/state.rs @@ -13,21 +13,34 @@ use super::{ }; use crate::{dbg, discovery, err, exchange, inf, wrn}; use ddm_api_types::db::RouterKind; -use ddm_protocol::v3::{PathVector, TunnelUpdate, UnderlayUpdate, Update}; +use ddm_api_types::net::TunnelOrigin; +use ddm_protocol::v3::{PathVector, TunnelUpdate, UnderlayUpdate}; +use ddm_protocol::v4::{MulticastPathHop, MulticastPathVector, Update}; use libnet::get_ipaddr_info; use slog::Logger; use std::collections::HashSet; use std::net::IpAddr; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::mpsc::Receiver; +use std::sync::mpsc::{Receiver, RecvTimeoutError}; use std::thread::{sleep, spawn}; -use std::time::Duration; +use std::time::{Duration, Instant}; use crate::discovery::Version; -use ddm_api_types::net::TunnelOrigin; use std::net::Ipv6Addr; +/// Cadence for the periodic pull in the [`Exchange`] state. The initial pull +/// is one-shot, so a neighbor that originates routes after we pull it, late +/// multicast group memberships for instance, would otherwise never be +/// imported absent a push from that neighbor. Pulling on this cadence +/// repairs that drift without operator intervention. It matches the +/// multicast sweep reconcile interval so both repair loops converge on the +/// same cadence. Pre-V4 peers are skipped, since their responses carry no +/// multicast half. The exchange loop checks a fixed deadline before +/// receiving another event, so a busy event queue cannot postpone the pull +/// indefinitely. +const EXCHANGE_RESYNC_INTERVAL: Duration = crate::mcast::RECONCILE_INTERVAL; + impl StateMachine { pub fn run(&mut self) -> Result<(), SmError> { let ctx = self.ctx.clone(); @@ -77,9 +90,8 @@ impl State for Init { wrn!( self.log, self.ctx.config.if_name, - "failed to get IPv6 address for interface {}: {}", + "failed to get IPv6 address for interface {}: {e}", &self.ctx.config.aobj_name, - e ); sleep(Duration::from_millis(self.ctx.config.ip_addr_wait)); continue; @@ -156,8 +168,7 @@ impl State for Solicit { err!( self.log, self.ctx.config.if_name, - "solicit event recv: {}", - e + "solicit event recv: {e}", ); continue; } @@ -169,6 +180,15 @@ impl State for Solicit { self.ctx.config.if_name, "transition solicit -> exchange" ); + + // The peer is now established on this link, so wake the + // multicast sweep for any of its groups whose import raced + // ahead of resolution. + crate::mcast::notify_peer_groups( + &self.ctx.db, + addr, + &self.ctx.mcast_notify, + ); return ( Box::new(Exchange::new( self.ctx.clone(), @@ -250,9 +270,10 @@ impl Exchange { version, rt.clone(), log.clone(), + exchange::UpdateMode::Redistribute, ) { sleep(Duration::from_millis(interval)); - wrn!(log, if_name, "exchange pull: {}", e); + wrn!(log, if_name, "exchange pull: {e}"); if stop.load(Ordering::Relaxed) { break; } @@ -260,6 +281,30 @@ impl Exchange { }); } + fn periodic_pull(&self) { + // The resync exists to repair multicast drift, and multicast first + // appears on the wire in V4. An earlier peer's response has no + // multicast half, so a pull would only replay the full underlay and + // tunnel tables while holding the route_update lock. + if self.version < Version::V4 { + return; + } + if let Err(e) = crate::exchange::pull( + self.ctx.clone(), + self.peer, + self.version, + self.ctx.rt.clone(), + self.log.clone(), + exchange::UpdateMode::ImportOnly, + ) { + wrn!( + self.log, + self.ctx.config.if_name, + "periodic exchange pull: {e}", + ); + } + } + fn wait_for_exchange_server_to_start(&self) { inf!( self.log, @@ -268,7 +313,7 @@ impl Exchange { ); let interval = 250; // TODO as parameter loop { - match exchange::do_pull( + match exchange::do_pull_v4( &self.ctx, &self.ctx.config.addr, &self.ctx.rt, @@ -293,13 +338,35 @@ impl Exchange { fn expire_peer( &mut self, - exchange_thread: &tokio::task::JoinHandle<()>, + exchange_handle: &exchange::ExchangeHandle, pull_stop: &AtomicBool, ) { - exchange_thread.abort(); + exchange_handle.abort(); self.ctx.iface.clear_peer(); - let (to_remove, to_remove_tnl) = - self.ctx.db.remove_nexthop_routes(self.peer); + self.withdraw_peer_routes(self.peer); + pull_stop.store(true, Ordering::Relaxed); + } + + /// Remove all routes imported via `peer`, clean up the forwarding state + /// derived from them, and, on transit routers, propagate withdraws to the + /// remaining peers. + /// + /// Called on peer expiry and on renumber, where a re-advertisement + /// carries a new peer address and `peer` is the prior one. + fn withdraw_peer_routes(&self, peer: Ipv6Addr) { + // Exchange updates take the same lock. If an old-peer update is + // already running, cleanup follows it and removes its imports. If + // cleanup wins, the update subsequently observes the changed peer + // identity and is discarded. + let _route_update = mg_common::lock!(self.ctx.iface.route_update); + let removed = self.ctx.db.remove_nexthop_routes(peer); + self.redistribute_removed_routes(&removed); + let crate::db::RemovedNexthopRoutes { + underlay: to_remove, + tunnel: to_remove_tnl, + multicast: to_remove_mcast, + .. + } = removed; let mut routes: Vec = Vec::new(); for x in &to_remove { let mut r: crate::sys::Route = x.clone().into(); @@ -325,21 +392,34 @@ impl Exchange { to_remove_tnl ); } - // if we're a transit router propagate withdraws for the - // expired peer. + + // The peer's routes are gone from the imported set, so we notify the + // multicast sweep of each affected underlay group. The sweep drops the + // peer's replication membership from DPD. + crate::mcast::notify_affected_groups( + to_remove_mcast.iter(), + &self.ctx.mcast_notify, + ); + } + + fn redistribute_removed_routes( + &self, + removed: &crate::db::RemovedNexthopRoutes, + ) { + // If we're a transit router propagate withdraws for the + // removed routes. if self.ctx.config.kind == RouterKind::Transit { dbg!( self.log, self.ctx.config.if_name, - "redistributing expire to {} peers", + "redistributing withdraws to {} peers", self.ctx.event_channels.len() ); - let underlay = if to_remove.is_empty() { - None - } else { - Some(UnderlayUpdate::withdraw( - to_remove + let underlay = (!removed.underlay.is_empty()).then(|| { + UnderlayUpdate::withdraw( + removed + .underlay .iter() .map(|x| PathVector { destination: x.destination, @@ -350,23 +430,61 @@ impl Exchange { }, }) .collect(), - )) - }; + ) + }); - let tunnel = if to_remove_tnl.is_empty() { + let tunnel = (!removed.tunnel.is_empty()).then(|| { + TunnelUpdate::withdraw( + removed.tunnel.iter().cloned().map(Into::into).collect(), + ) + }); + + // Downstream peers collapse all paths through us into one route. + // For each removed origin, either withdraw the final path or + // refresh the peer with a remaining imported/local path. + let multicast = if removed.multicast.is_empty() { None } else { - Some(TunnelUpdate::withdraw( - to_remove_tnl.iter().cloned().map(Into::into).collect(), - )) + let withdrawals: HashSet<_> = removed + .multicast + .iter() + .map(|route| MulticastPathVector { + origin: (&route.origin).into(), + path: Vec::new(), + }) + .collect(); + let update = crate::exchange::reconcile_multicast_withdrawals( + &withdrawals, + &removed.mcast_reachability, + &MulticastPathHop::new( + self.ctx.hostname.clone(), + self.ctx.config.addr, + ), + ); + if update.announce.is_empty() && update.withdraw.is_empty() { + None + } else { + Some(update) + } }; - let push = Update { underlay, tunnel }; + let push = Arc::new(Update { + underlay, + tunnel, + multicast, + }); for ec in &self.ctx.event_channels { - ec.send(Event::Peer(PeerEvent::Push(push.clone()))).unwrap(); + if let Err(e) = + ec.send(Event::Peer(PeerEvent::Push(Arc::clone(&push)))) + { + err!( + self.log, + self.ctx.config.if_name, + "deliver redistributed withdraw: {e}", + ); + } } } - pull_stop.store(true, Ordering::Relaxed); } } @@ -376,7 +494,7 @@ impl State for Exchange { event: Receiver, ) -> (Box, Receiver) { self.ctx.iface.transition(FsmState::Exchange); - let exchange_thread = loop { + let exchange_handle = loop { match exchange::handler( self.ctx.clone(), self.ctx.config.addr, @@ -406,15 +524,23 @@ impl State for Exchange { // loop below. self.initial_pull(pull_stop.clone()); + let mut resync_deadline = Instant::now() + EXCHANGE_RESYNC_INTERVAL; loop { - let e = match event.recv() { + if Instant::now() >= resync_deadline { + self.periodic_pull(); + resync_deadline = Instant::now() + EXCHANGE_RESYNC_INTERVAL; + } + + let wait = + resync_deadline.saturating_duration_since(Instant::now()); + let e = match event.recv_timeout(wait) { Ok(e) => e, + Err(RecvTimeoutError::Timeout) => continue, Err(e) => { err!( self.log, self.ctx.config.if_name, - "exchange event recv: {}", - e + "exchange event recv: {e}", ); continue; } @@ -442,8 +568,7 @@ impl State for Exchange { err!( self.log, self.ctx.config.if_name, - "announce: {}", - e, + "announce: {e}", ); wrn!( self.log, @@ -451,7 +576,7 @@ impl State for Exchange { "expiring peer {} due to failed announce", self.peer, ); - self.expire_peer(&exchange_thread, &pull_stop); + self.expire_peer(&exchange_handle, &pull_stop); return ( Box::new(Solicit::new( self.ctx.clone(), @@ -477,8 +602,7 @@ impl State for Exchange { err!( self.log, self.ctx.config.if_name, - "announce tunnel: {}", - e, + "announce tunnel: {e}", ); wrn!( self.log, @@ -486,7 +610,7 @@ impl State for Exchange { "expiring peer {} due to failed tunnel announce", self.peer, ); - self.expire_peer(&exchange_thread, &pull_stop); + self.expire_peer(&exchange_handle, &pull_stop); return ( Box::new(Solicit::new( self.ctx.clone(), @@ -518,8 +642,7 @@ impl State for Exchange { err!( self.log, self.ctx.config.if_name, - "withdraw: {}", - e, + "withdraw: {e}", ); wrn!( self.log, @@ -527,7 +650,7 @@ impl State for Exchange { "expiring peer {} due to failed withdraw", self.peer, ); - self.expire_peer(&exchange_thread, &pull_stop); + self.expire_peer(&exchange_handle, &pull_stop); return ( Box::new(Solicit::new( self.ctx.clone(), @@ -553,8 +676,7 @@ impl State for Exchange { err!( self.log, self.ctx.config.if_name, - "withdraw tunnel: {}", - e, + "withdraw tunnel: {e}", ); wrn!( self.log, @@ -562,7 +684,146 @@ impl State for Exchange { "expiring peer {} due to failed tunnel withdraw", self.peer, ); - self.expire_peer(&exchange_thread, &pull_stop); + self.expire_peer(&exchange_handle, &pull_stop); + return ( + Box::new(Solicit::new( + self.ctx.clone(), + self.log.clone(), + )), + event, + ); + } + } + Event::Admin(AdminEvent::AnnounceMulticast(groups)) => { + // Build a `MulticastPathVector` for each origin, recording + // our hop in the path. + let hop = MulticastPathHop::new( + self.ctx.hostname.clone(), + self.ctx.config.addr, + ); + let path_vectors: HashSet<_> = groups + .iter() + .map(|origin| { + ddm_api_types::exchange::MulticastPathVector { + origin: origin.into(), + path: vec![hop.clone()], + } + }) + .collect(); + + if let Err(e) = crate::exchange::announce_multicast( + &self.ctx, + self.ctx.config.clone(), + path_vectors, + self.peer, + self.version, + self.ctx.rt.clone(), + self.log.clone(), + ) { + err!( + self.log, + self.ctx.config.if_name, + "announce multicast: {e}", + ); + wrn!( + self.log, + self.ctx.config.if_name, + "expiring peer {} due to failed multicast announce", + self.peer, + ); + self.expire_peer(&exchange_handle, &pull_stop); + return ( + Box::new(Solicit::new( + self.ctx.clone(), + self.log.clone(), + )), + event, + ); + } + } + Event::Admin(AdminEvent::WithdrawMulticast(origins)) => { + // The persistent local origins were removed by the + // modification that preceded this event. The reachability + // snapshot is read here, at processing time, so an + // import that raced the admin request is observed and + // produces a replacement announcement rather than a + // stale final withdrawal. Whenever an imported path to + // an origin remains, replace the local announcement with + // that path. Otherwise, propagate the final withdrawal. + let hop = MulticastPathHop::new( + self.ctx.hostname.clone(), + self.ctx.config.addr, + ); + let withdrawals: HashSet<_> = origins + .iter() + .map(|origin| MulticastPathVector { + origin: origin.into(), + path: Vec::new(), + }) + .collect(); + let reachability = self.ctx.db.multicast_reachability(); + let update = + crate::exchange::reconcile_multicast_withdrawals( + &withdrawals, + &reachability, + &hop, + ); + + if !update.announce.is_empty() + && let Err(e) = crate::exchange::announce_multicast( + &self.ctx, + self.ctx.config.clone(), + update.announce, + self.peer, + self.version, + self.ctx.rt.clone(), + self.log.clone(), + ) + { + err!( + self.log, + self.ctx.config.if_name, + "replace withdrawn multicast path: {e}", + ); + wrn!( + self.log, + self.ctx.config.if_name, + "expiring peer {} due to failed multicast replacement", + self.peer, + ); + self.expire_peer(&exchange_handle, &pull_stop); + return ( + Box::new(Solicit::new( + self.ctx.clone(), + self.log.clone(), + )), + event, + ); + } + + if !update.withdraw.is_empty() + && let Err(e) = crate::exchange::withdraw_multicast( + &self.ctx, + self.ctx.config.clone(), + update.withdraw, + self.peer, + self.version, + self.ctx.rt.clone(), + self.log.clone(), + ) + { + err!( + self.log, + self.ctx.config.if_name, + "withdraw multicast: {e}", + ); + wrn!( + self.log, + self.ctx.config.if_name, + "expiring peer {} due to failed multicast withdraw", + self.peer, + ); + self.expire_peer(&exchange_handle, &pull_stop); return ( Box::new(Solicit::new( self.ctx.clone(), @@ -580,7 +841,7 @@ impl State for Exchange { "administratively expiring peer {}", peer, ); - self.expire_peer(&exchange_thread, &pull_stop); + self.expire_peer(&exchange_handle, &pull_stop); return ( Box::new(Solicit::new( self.ctx.clone(), @@ -597,12 +858,12 @@ impl State for Exchange { self.version, self.ctx.rt.clone(), self.log.clone(), + exchange::UpdateMode::Redistribute, ) { err!( self.log, self.ctx.config.if_name, - "exchange pull: {}", - e + "exchange pull: {e}", ); } } @@ -614,6 +875,8 @@ impl State for Exchange { self.peer, update, ); + let update = Arc::try_unwrap(update) + .unwrap_or_else(|arc| (*arc).clone()); if let Some(push) = update.underlay { if !push.announce.is_empty() && let Err(e) = crate::exchange::announce_underlay( @@ -629,8 +892,7 @@ impl State for Exchange { err!( self.log, self.ctx.config.if_name, - "announce: {}", - e, + "announce: {e}", ); wrn!( self.log, @@ -638,7 +900,7 @@ impl State for Exchange { "expiring peer {} due to failed announce", self.peer, ); - self.expire_peer(&exchange_thread, &pull_stop); + self.expire_peer(&exchange_handle, &pull_stop); return ( Box::new(Solicit::new( self.ctx.clone(), @@ -647,6 +909,7 @@ impl State for Exchange { event, ); } + if !push.withdraw.is_empty() && let Err(e) = crate::exchange::withdraw_underlay( &self.ctx, @@ -661,8 +924,7 @@ impl State for Exchange { err!( self.log, self.ctx.config.if_name, - "withdraw: {}", - e, + "withdraw: {e}", ); wrn!( self.log, @@ -670,7 +932,73 @@ impl State for Exchange { "expiring peer {} due to failed withdraw", self.peer, ); - self.expire_peer(&exchange_thread, &pull_stop); + self.expire_peer(&exchange_handle, &pull_stop); + return ( + Box::new(Solicit::new( + self.ctx.clone(), + self.log.clone(), + )), + event, + ); + } + } + + if let Some(push) = update.multicast { + if !push.announce.is_empty() + && let Err(e) = crate::exchange::announce_multicast( + &self.ctx, + self.ctx.config.clone(), + push.announce, + self.peer, + self.version, + self.ctx.rt.clone(), + self.log.clone(), + ) + { + err!( + self.log, + self.ctx.config.if_name, + "announce multicast: {e}", + ); + wrn!( + self.log, + self.ctx.config.if_name, + "expiring peer {} due to failed multicast announce", + self.peer, + ); + self.expire_peer(&exchange_handle, &pull_stop); + return ( + Box::new(Solicit::new( + self.ctx.clone(), + self.log.clone(), + )), + event, + ); + } + + if !push.withdraw.is_empty() + && let Err(e) = crate::exchange::withdraw_multicast( + &self.ctx, + self.ctx.config.clone(), + push.withdraw, + self.peer, + self.version, + self.ctx.rt.clone(), + self.log.clone(), + ) + { + err!( + self.log, + self.ctx.config.if_name, + "withdraw multicast: {e}", + ); + wrn!( + self.log, + self.ctx.config.if_name, + "expiring peer {} due to failed multicast withdraw", + self.peer, + ); + self.expire_peer(&exchange_handle, &pull_stop); return ( Box::new(Solicit::new( self.ctx.clone(), @@ -688,7 +1016,7 @@ impl State for Exchange { "expiring peer {} due to discovery event", self.peer, ); - self.expire_peer(&exchange_thread, &pull_stop); + self.expire_peer(&exchange_handle, &pull_stop); return ( Box::new(Solicit::new( self.ctx.clone(), @@ -704,15 +1032,41 @@ impl State for Exchange { "expiring peer {} due to failed solicit", self.peer, ); - self.expire_peer(&exchange_thread, &pull_stop); + self.expire_peer(&exchange_handle, &pull_stop); return ( Box::new(Init::new(self.ctx.clone(), self.log.clone())), event, ); } Event::Neighbor(NeighborEvent::Advertise((addr, version))) => { + if addr != self.peer { + // A re-advertisement carrying a new address renumbers + // the peer. Expiry removes routes keyed on the + // current address only, so routes under the prior + // address would otherwise persist indefinitely. + // Withdraw them as if that address expired. + inf!( + self.log, + self.ctx.config.if_name, + "peer renumbered from {} to {addr}", + self.peer, + ); + // Rebind the running exchange handler so that + // pushes arriving under the new address cannot + // recreate routes keyed by the prior peer after + // cleanup. + exchange_handle.renumber_peer(addr); + self.withdraw_peer_routes(self.peer); + } self.peer = addr; self.version = version; + // Wake the multicast sweep for the peer's groups under + // the advertised address. + crate::mcast::notify_peer_groups( + &self.ctx.db, + addr, + &self.ctx.mcast_notify, + ); } } } diff --git a/ddm/src/sys.rs b/ddm/src/sys.rs index c251ab600..ee9cdee49 100644 --- a/ddm/src/sys.rs +++ b/ddm/src/sys.rs @@ -8,6 +8,7 @@ use ddm_api_types::db::TunnelRoute; use dpd_client::Client; use dpd_client::ClientState; use dpd_client::types; +use mg_common::tfport::{TfportKind, parse_tfport_name, tfport_port_id}; use oxnet::{IpNet, Ipv4Net, Ipv6Net}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -23,7 +24,9 @@ use ::{ std::collections::HashMap, }; -const DDM_DPD_TAG: &str = "ddmd"; +/// Client identity tag carried in `dpd_client::ClientState` to identify +/// `ddmd` to DPD, distinct from any per-group authorization tag DPD owns. +pub(crate) const DDM_DPD_TAG: &str = "ddmd"; #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] pub struct Route { @@ -176,25 +179,25 @@ pub fn add_routes_dendrite( } }; - // TODO this is gross, use link type properties rather than futzing - // around with strings. - let Some(egress_port_num) = ifname - .strip_prefix("tfportrear") - .and_then(|x| x.strip_suffix("_0")) - .map(|x| x.trim()) - .and_then(|x| x.parse::().ok()) - else { - err!(log, ifname, "expected tfportrear"); - continue; + let tfport = match parse_tfport_name(ifname) { + Ok(tfport) => tfport, + Err(e) => { + err!(log, ifname, "{e}"); + continue; + } }; // TODO this assumes ddm only operates on rear ports, which will not be // true for multi-rack deployments. - let port_name = format!("rear{}", egress_port_num); - let port_id = match types::Rear::try_from(&port_name) { - Ok(rear) => PortId::Rear(rear), + if tfport.kind != TfportKind::Rear { + err!(log, ifname, "expected rear tfport, got {:?}", tfport.kind); + continue; + } + + let port_id = match tfport_port_id(tfport.kind, tfport.port) { + Ok(port_id) => port_id, Err(e) => { - err!(log, ifname, "bad port name ({port_name}): {e}"); + err!(log, ifname, "{e}"); continue; } }; @@ -206,11 +209,10 @@ pub fn add_routes_dendrite( r.dest, r.gw, port_id, - 0, + tfport.link, ); - // TODO breakout considerations - let link_id = types::LinkId(0); + let link_id = types::LinkId(tfport.link); let target = types::Ipv6Route { tag: DDM_DPD_TAG.into(), diff --git a/ddmadm/src/main.rs b/ddmadm/src/main.rs index 21d56fdf3..820fa0f79 100644 --- a/ddmadm/src/main.rs +++ b/ddmadm/src/main.rs @@ -6,7 +6,7 @@ use anyhow::Result; use clap::Parser; use colored::*; use ddm_admin_client::Client; -use ddm_api_types_versions::latest::db::PeerStatus; +use ddm_api_types_versions::latest::db::{PeerStatus, RouterKind}; use ddm_api_types_versions::latest::net as types; use mg_common::cli::oxide_cli_style; use mg_common::format_duration_human; @@ -63,6 +63,18 @@ enum SubCommand { /// Withdraw prefixes from a DDM router. TunnelWithdraw(TunnelEndpoint), + /// Get multicast groups imported from DDM peers. + MulticastImported, + + /// Get locally originated multicast groups. + MulticastOriginated, + + /// Advertise multicast groups from this router. + MulticastAdvertise(MulticastGroup), + + /// Withdraw multicast groups from this router. + MulticastWithdraw(MulticastGroup), + /// Sync prefix information from peers. Sync, } @@ -87,6 +99,29 @@ struct TunnelEndpoint { pub metric: u64, } +#[derive(Debug, Parser)] +struct MulticastGroup { + /// Overlay multicast group address (e.g. 233.252.0.1 or ff0e::1). + #[arg(short = 'g', long)] + pub overlay_group: IpAddr, + + /// Underlay multicast address (ff04::/64 admin-local scope). + #[arg(short = 'u', long)] + pub underlay_group: Ipv6Addr, + + /// Virtual Network Identifier. + #[arg(short, long)] + pub vni: u32, + + /// Path metric. + #[arg(short, long, default_value_t = 0)] + pub metric: u64, + + /// Source address for (S,G) routes (omit for (*,G)). + #[arg(short, long)] + pub source: Option, +} + #[derive(Debug, Parser)] struct Peer { addr: Ipv6Addr, @@ -144,8 +179,8 @@ async fn run() -> Result<()> { info.host, info.addr, match info.kind { - ddm_api_types_versions::latest::db::RouterKind::Server => "Server", - ddm_api_types_versions::latest::db::RouterKind::Transit => "Transit", + RouterKind::Server => "Server", + RouterKind::Transit => "Transit", }, state, format_duration_human(*duration), @@ -262,6 +297,107 @@ async fn run() -> Result<()> { }]) .await?; } + SubCommand::MulticastImported => { + let msg = client.get_multicast_groups().await?; + let mut routes: Vec<_> = msg.into_inner().into_iter().collect(); + routes.sort_by(|a, b| { + a.origin + .overlay_group + .cmp(&b.origin.overlay_group) + .then_with(|| a.origin.source.cmp(&b.origin.source)) + }); + let mut tw = TabWriter::new(stdout()); + writeln!( + &mut tw, + "{}\t{}\t{}\t{}\t{}\t{}", + "Overlay Group".dimmed(), + "Underlay Group".dimmed(), + "VNI".dimmed(), + "Metric".dimmed(), + "Source".dimmed(), + "Path".dimmed(), + )?; + for route in &routes { + let source = match &route.origin.source { + Some(s) => s.to_string(), + None => "(*,G)".to_string(), + }; + let path: Vec<_> = route + .path + .iter() + .rev() + .map(|h| h.router_id.clone()) + .collect(); + writeln!( + &mut tw, + "{}\t{}\t{}\t{}\t{}\t{}", + route.origin.overlay_group, + route.origin.underlay_group, + route.origin.vni, + route.origin.metric, + source, + path.join(" "), + )?; + } + tw.flush()?; + } + SubCommand::MulticastOriginated => { + let msg = client.get_originated_multicast_groups().await?; + let mut origins: Vec<_> = msg.into_inner().into_iter().collect(); + origins.sort_by(|a, b| { + a.overlay_group + .cmp(&b.overlay_group) + .then_with(|| a.source.cmp(&b.source)) + }); + let mut tw = TabWriter::new(stdout()); + writeln!( + &mut tw, + "{}\t{}\t{}\t{}\t{}", + "Overlay Group".dimmed(), + "Underlay Group".dimmed(), + "VNI".dimmed(), + "Metric".dimmed(), + "Source".dimmed(), + )?; + for origin in &origins { + let source = match &origin.source { + Some(s) => s.to_string(), + None => "(*,G)".to_string(), + }; + writeln!( + &mut tw, + "{}\t{}\t{}\t{}\t{}", + origin.overlay_group, + origin.underlay_group, + u32::from(origin.vni), + origin.metric, + source, + )?; + } + tw.flush()?; + } + SubCommand::MulticastAdvertise(mg) => { + client + .advertise_multicast_groups(&vec![types::MulticastOrigin { + overlay_group: mg.overlay_group.try_into()?, + underlay_group: mg.underlay_group.try_into()?, + vni: types::Vni::try_from(mg.vni)?, + metric: mg.metric, + source: mg.source, + }]) + .await?; + } + SubCommand::MulticastWithdraw(mg) => { + client + .withdraw_multicast_groups(&vec![types::MulticastOrigin { + overlay_group: mg.overlay_group.try_into()?, + underlay_group: mg.underlay_group.try_into()?, + vni: types::Vni::try_from(mg.vni)?, + metric: mg.metric, + source: mg.source, + }]) + .await?; + } SubCommand::Sync => { client.sync().await?; } diff --git a/ddmd/src/main.rs b/ddmd/src/main.rs index 671e5cdd1..e957ef1ef 100644 --- a/ddmd/src/main.rs +++ b/ddmd/src/main.rs @@ -108,9 +108,9 @@ struct Arg { sled_uuid: Option, /// Serve only the admin API. Skips the routing state machine - /// (discovery, exchange, route synchronization), allowing test fixtures - /// to obtain a real `ddmd` admin endpoint without the kernel-level - /// networking the state machine requires. + /// (discovery, exchange, route synchronization), allowing a real `ddmd` + /// admin endpoint without the kernel-level networking the state machine + /// requires. /// /// Analogous to `mgd --no-bgp-dispatcher`. #[arg(long, default_value_t = false, conflicts_with = "addr")] @@ -160,14 +160,34 @@ async fn run() { .to_string_lossy() .to_string(); + // Notify channel into the multicast membership sweep. Each state machine's + // context holds the sender and signals a group's address when its imported + // membership changes. The sweep started by start_mcast_sweep owns the + // receiver and wakes early to reconcile the full tracked set. + let (notify_tx, notify_rx) = + tokio::sync::mpsc::channel::(ddm::MCAST_NOTIFY_CHANNEL_DEPTH); + let (sms, event_channels) = - start_state_machines(&arg, &db, &dpd, &hostname, &rt, &log); + start_state_machines(&arg, &db, &dpd, &hostname, &rt, ¬ify_tx, &log); termination_handler(db.clone(), dpd.clone(), rt.clone(), log.clone()); let router_stats = Arc::new(RouterStats::default()); + // Per-interface state machine contexts shared between the multicast sweep + // and the admin context, seeded from the running state machines. + // + // Under --api-only there are no state machines, so the set is empty. let peers: Vec = sms.iter().map(|x| x.ctx.clone()).collect(); + start_mcast_sweep( + notify_rx, + dpd.clone(), + db.clone(), + peers.clone(), + rt.clone(), + log.clone(), + ); + let stats_handler = if arg.with_stats { if let (Some(rack_uuid), Some(sled_uuid)) = (arg.rack_uuid, arg.sled_uuid) @@ -233,6 +253,7 @@ fn start_state_machines( dpd: &Option, hostname: &str, rt: &Arc, + notify_tx: &tokio::sync::mpsc::Sender, log: &Logger, ) -> ( Vec, @@ -273,6 +294,7 @@ fn start_state_machines( rt: rt.clone(), iface: Arc::new(InterfaceState::default()), stats: Arc::new(ddm::sm::SessionStats::default()), + mcast_notify: notify_tx.clone(), }; let sm = StateMachine { ctx, rx: Some(rx) }; @@ -309,6 +331,7 @@ fn start_state_machines( _dpd: &Option, _hostname: &str, _rt: &Arc, + _notify_tx: &tokio::sync::mpsc::Sender, _log: &Logger, ) -> ( Vec, @@ -317,7 +340,51 @@ fn start_state_machines( (Vec::new(), Vec::new()) } -/// Install a Ctrl-C handler that withdraws ddmd's imported routes from the +/// Spawn the underlay multicast membership sweep, the multicast analog of the +/// unicast import-to-DPD path `ddmd` already performs in-process. The sweep +/// runs as a task on the daemon's runtime, reconciling every tracked group on +/// each pass, woken early by a trigger and otherwise self-ticking on a fixed +/// interval. +/// +/// Takes `notify_rx` by value so any path that does not start the sweep drops +/// it, closing the channel and making the state machines' notify sends fail +/// fast rather than accumulate as unread. +#[cfg(all(feature = "backend", target_os = "illumos"))] +fn start_mcast_sweep( + notify_rx: tokio::sync::mpsc::Receiver, + dpd: Option, + db: Db, + peers: Vec, + rt: Arc, + log: Logger, +) { + let Some(dpd) = dpd else { + // No backend: returning drops notify_rx and closes the channel. + return; + }; + // Under --api-only there are no state machines, so no peers or imports + // exist to derive membership from. A sweep would compute an empty + // desired set for every seeded group and drain its DPD members. + if peers.is_empty() { + return; + } + rt.spawn(ddm::mcast::run(db, peers, dpd, notify_rx, log)); +} + +/// Non-illumos variant: underlay multicast replicates only on a switch, so the +/// sweep never starts. Consuming `notify_rx` drops it, closing the channel. +#[cfg(not(all(feature = "backend", target_os = "illumos")))] +fn start_mcast_sweep( + _notify_rx: tokio::sync::mpsc::Receiver, + _dpd: Option, + _db: Db, + _peers: Vec, + _rt: Arc, + _log: Logger, +) { +} + +/// Install a Ctrl-C handler that withdraws `ddmd`'s imported routes from the /// kernel before exiting. On non-illumos builds there are no kernel routes /// to withdraw, so the handler just exits cleanly. fn termination_handler( diff --git a/mg-api-types/versions/src/impls/mrib.rs b/mg-api-types/versions/src/impls/mrib.rs index 11a7b9dca..593abcf25 100644 --- a/mg-api-types/versions/src/impls/mrib.rs +++ b/mg-api-types/versions/src/impls/mrib.rs @@ -2,8 +2,6 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -// Copyright 2026 Oxide Computer Company - //! Proptest `Arbitrary` impls and strategy helpers for the latest MRIB types. use std::net::{Ipv4Addr, Ipv6Addr}; diff --git a/mg-api-types/versions/src/multicast_support/mod.rs b/mg-api-types/versions/src/multicast_support/mod.rs index 5f0b1552b..76d666354 100644 --- a/mg-api-types/versions/src/multicast_support/mod.rs +++ b/mg-api-types/versions/src/multicast_support/mod.rs @@ -2,8 +2,6 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -// Copyright 2026 Oxide Computer Company - //! Version `MULTICAST_SUPPORT` of the Maghemite Admin API. //! //! Adds MRIB (Multicast Routing Information Base) endpoints for static diff --git a/mg-api-types/versions/src/multicast_support/mrib.rs b/mg-api-types/versions/src/multicast_support/mrib.rs index fe586701f..bc21fa4d1 100644 --- a/mg-api-types/versions/src/multicast_support/mrib.rs +++ b/mg-api-types/versions/src/multicast_support/mrib.rs @@ -14,7 +14,6 @@ use std::fmt::{self, Formatter}; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; -use std::str::FromStr; use chrono::{DateTime, Utc}; use client_common::address::{ @@ -22,23 +21,15 @@ use client_common::address::{ IPV4_SSM_RESERVED_SUBNET, IPV6_MULTICAST_RANGE, UNDERLAY_MULTICAST_SUBNET, is_ssm_address, }; +use client_common::multicast::UnderlayMulticastError; +pub use client_common::multicast::UnderlayMulticastIpv6; +use client_common::vni::VniError; +pub use client_common::vni::{DEFAULT_MULTICAST_VNI, MAX_VNI, Vni}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use crate::v1::rdb::rib::AddressFamily; -/// Maximum Geneve VNI value. -/// -/// Virtual Network Identifiers are constrained to 24-bit values per the -/// Geneve specification (RFC 8926 Section 3.3). -pub const MAX_VNI: u32 = 0xFF_FFFF; - -/// Default VNI for fleet-wide multicast routing. -/// -/// A low-numbered VNI chosen to avoid colliding with user VNIs, though it is -/// not yet within the Oxide-reserved range. -pub const DEFAULT_MULTICAST_VNI: u32 = 77; - /// Errors raised while validating or serializing multicast route data. #[derive(thiserror::Error, Debug)] pub enum MulticastError { @@ -55,79 +46,28 @@ pub enum MulticastError { DbKey(String), } -/// A validated Geneve Virtual Network Identifier. -/// -/// Wraps a 24-bit VNI, rejecting any value above [`MAX_VNI`] at construction -/// and deserialization so an out-of-range identifier is unrepresentable. -#[derive( - Debug, - Copy, - Clone, - Eq, - PartialEq, - PartialOrd, - Ord, - Hash, - Serialize, - Deserialize, - JsonSchema, -)] -#[serde(try_from = "u32", into = "u32")] -#[schemars(transparent)] -pub struct Vni(u32); - -impl Vni { - /// Default VNI for fleet-wide multicast routing. - pub const DEFAULT_MULTICAST: Self = Self(DEFAULT_MULTICAST_VNI); - - /// Create a validated VNI. - /// - /// # Errors - /// - /// Returns [`MulticastError::Validation`] if `value` exceeds [`MAX_VNI`], - /// the largest 24-bit Geneve VNI. - /// - /// # Examples - /// - /// ``` - /// use mg_api_types_versions::latest::mrib::{MAX_VNI, Vni}; - /// - /// assert!(Vni::new(77).is_ok()); - /// assert!(Vni::new(MAX_VNI + 1).is_err()); - /// ``` - pub fn new(value: u32) -> Result { - if value > MAX_VNI { - return Err(MulticastError::Validation(format!( - "VNI {value} exceeds the maximum 24-bit value {MAX_VNI}" - ))); +// Bridge errors from the shared `client_common` newtypes so call sites using +// `?` with `UnderlayMulticastIpv6::new` or `Vni::new` continue to surface +// validation failures through `MulticastError`. +impl From for MulticastError { + fn from(value: UnderlayMulticastError) -> Self { + match value { + UnderlayMulticastError::NotInSubnet { addr } => { + MulticastError::Validation(format!( + "underlay address {addr} is not within \ + UNDERLAY_MULTICAST_SUBNET" + )) + } + UnderlayMulticastError::InvalidIpv6(e) => { + MulticastError::Validation(format!("invalid IPv6 address: {e}")) + } } - Ok(Self(value)) - } - - /// Return the underlying 24-bit value. - #[inline] - pub const fn as_u32(self) -> u32 { - self.0 } } -impl fmt::Display for Vni { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.0) - } -} - -impl TryFrom for Vni { - type Error = MulticastError; - - fn try_from(value: u32) -> Result { - Self::new(value) - } -} - -impl From for u32 { - fn from(vni: Vni) -> Self { - vni.0 +impl From for MulticastError { + fn from(value: VniError) -> Self { + MulticastError::Validation(value.to_string()) } } @@ -562,92 +502,6 @@ impl From for Ipv6Addr { } } -/// A validated underlay multicast IPv6 address within ff04::/64. -/// -/// The Oxide rack maps overlay multicast groups 1:1 to admin-local scoped -/// IPv6 multicast addresses in `UNDERLAY_MULTICAST_SUBNET` (ff04::/64). -/// This type enforces that invariant at construction time. -// TODO: Duplicates `dpd_types::mcast::UnderlayMulticastIpv6` in dendrite. -// Both should be consolidated into `oxnet`, the cycle-free leaf crate that -// maghemite, dendrite, and omicron already share. -#[derive( - Debug, - Copy, - Clone, - Eq, - PartialEq, - PartialOrd, - Ord, - Hash, - Serialize, - Deserialize, - JsonSchema, -)] -#[serde(try_from = "Ipv6Addr", into = "Ipv6Addr")] -#[schemars(transparent)] -pub struct UnderlayMulticastIpv6(Ipv6Addr); - -impl UnderlayMulticastIpv6 { - /// Create a new validated underlay multicast address. - /// - /// # Errors - /// - /// Returns an error if the address is not within `UNDERLAY_MULTICAST_SUBNET` - /// (ff04::/64). - pub fn new(value: Ipv6Addr) -> Result { - if !UNDERLAY_MULTICAST_SUBNET.contains(value) { - return Err(MulticastError::Validation(format!( - "underlay address {value} is not within \ - {UNDERLAY_MULTICAST_SUBNET}" - ))); - } - Ok(Self(value)) - } - - /// Returns the underlying IPv6 address. - #[inline] - pub const fn ip(&self) -> Ipv6Addr { - self.0 - } -} - -impl fmt::Display for UnderlayMulticastIpv6 { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.0) - } -} - -impl TryFrom for UnderlayMulticastIpv6 { - type Error = MulticastError; - - fn try_from(value: Ipv6Addr) -> Result { - Self::new(value) - } -} - -impl From for Ipv6Addr { - fn from(addr: UnderlayMulticastIpv6) -> Self { - addr.0 - } -} - -impl From for IpAddr { - fn from(addr: UnderlayMulticastIpv6) -> Self { - IpAddr::V6(addr.0) - } -} - -impl FromStr for UnderlayMulticastIpv6 { - type Err = MulticastError; - - fn from_str(s: &str) -> Result { - let addr: Ipv6Addr = s.parse().map_err(|_| { - MulticastError::Validation(format!("invalid IPv6 address: {s}")) - })?; - Self::new(addr) - } -} - /// A validated multicast group address (IPv4 or IPv6). /// /// This type guarantees that the contained address is a routable multicast @@ -1141,32 +995,15 @@ pub enum MulticastSourceProtocol { #[cfg(test)] mod tests { - use omicron_common::api::external::Vni as CanonicalVni; - use super::*; - /// Assert the locally copied VNI literals equal their - /// `omicron_common::api::external::Vni` originals so they cannot drift. - /// - /// `omicron_common` is a dev-dependency only, so it does not appear in the - /// normal dependency tree the no-omicron CI check inspects. - #[test] - fn vni_constants_match_canonical_values() { - assert_eq!(MAX_VNI, CanonicalVni::MAX_VNI); - assert_eq!( - DEFAULT_MULTICAST_VNI, - CanonicalVni::DEFAULT_MULTICAST_VNI.as_u32() - ); - } - - /// The [`Vni`] newtype accepts in-range values and rejects values above - /// [`MAX_VNI`], enforcing the 24-bit invariant at construction. #[test] - fn vni_rejects_out_of_range() { - assert_eq!(Vni::new(0).unwrap().as_u32(), 0); - assert_eq!(Vni::new(MAX_VNI).unwrap().as_u32(), MAX_VNI); - assert_eq!(Vni::DEFAULT_MULTICAST.as_u32(), DEFAULT_MULTICAST_VNI); - assert!(Vni::new(MAX_VNI + 1).is_err()); - assert!(Vni::new(u32::MAX).is_err()); + fn unicast_rejects_non_routable_sources() { + // 0/8 "this network" (RFC 791). + assert!(UnicastAddrV4::new(Ipv4Addr::new(0, 1, 2, 3)).is_err()); + // Link-local, 169.254/16 (RFC 3927): not router-forwarded. + assert!(UnicastAddrV4::new(Ipv4Addr::new(169, 254, 0, 1)).is_err()); + // Link-local, fe80::/10 (RFC 4291): not forwarded. + assert!(UnicastAddrV6::new("fe80::1".parse().unwrap()).is_err()); } } diff --git a/mg-common/Cargo.toml b/mg-common/Cargo.toml index ad4218def..501e22f4e 100644 --- a/mg-common/Cargo.toml +++ b/mg-common/Cargo.toml @@ -7,6 +7,7 @@ edition = "2024" clap.workspace = true anyhow.workspace = true ddm-api-types.workspace = true +dpd-client.workspace = true anstyle.workspace = true serde.workspace = true schemars.workspace = true @@ -26,6 +27,10 @@ libc.workspace = true workspace = true optional = true +[dev-dependencies] +proptest.workspace = true +serde_json.workspace = true + [features] default = ["libnet"] libnet = ["dep:libnet"] diff --git a/mg-common/src/lib.rs b/mg-common/src/lib.rs index b2af8cd22..abb47e2fb 100644 --- a/mg-common/src/lib.rs +++ b/mg-common/src/lib.rs @@ -8,6 +8,7 @@ pub mod nexus; pub mod smf; pub mod stats; pub mod test; +pub mod tfport; pub mod thread; use std::time::Duration; diff --git a/mg-common/src/tfport.rs b/mg-common/src/tfport.rs new file mode 100644 index 000000000..b758b181f --- /dev/null +++ b/mg-common/src/tfport.rs @@ -0,0 +1,269 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Parsing of Tofino port (`tfport`) datalink names. +//! +//! Tofino switch ports are surfaced to the host as illumos datalinks named +//! `tfport_[.vlan]`; for example, `tfportqsfp10_0` (front +//! panel) or `tfportrear0_0.100` (backplane). This module parses that form +//! into its components so callers can map a datalink name back to a switch +//! port without each one hand-rolling its own string handling. + +/// Prefix shared by every `tfport` datalink name. +const TFPORT_DEVICE_PREFIX: &str = "tfport"; + +/// Switch-port device kind encoded in a `tfport` datalink name. +/// +/// Front-panel links typically appear as `qsfp`. Backplane links toward other +/// sleds (the multicast underlay path) typically appear as `rear`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TfportKind { + Qsfp, + Rear, +} + +impl TfportKind { + /// Device token as it appears both in a tfport name (`tfportrear0_0`) and + /// in a dpd port name (`rear0`). + pub fn token(self) -> &'static str { + match self { + TfportKind::Qsfp => "qsfp", + TfportKind::Rear => "rear", + } + } + + /// Parse a device kind from its datalink token (`qsfp`, `rear`). + pub fn from_token(s: &str) -> Option { + match s { + "qsfp" => Some(TfportKind::Qsfp), + "rear" => Some(TfportKind::Rear), + _ => None, + } + } +} + +/// Components parsed from a `tfport` datalink name. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TfportName { + /// Device kind (front-panel vs backplane). + pub kind: TfportKind, + /// Switch port number within the kind. + pub port: u8, + /// Link (lane) number within the port. + pub link: u8, + /// Optional VLAN tag appended after a `.`. + pub vlan: Option, +} + +/// Parse a `tfport` datalink name into its components. +/// +/// # Examples +/// +/// ``` +/// use mg_common::tfport::{parse_tfport_name, TfportKind, TfportName}; +/// assert_eq!( +/// parse_tfport_name("tfportqsfp10_0.100").unwrap(), +/// TfportName { kind: TfportKind::Qsfp, port: 10, link: 0, vlan: Some(100) }, +/// ); +/// assert_eq!( +/// parse_tfport_name("tfportrear0_0").unwrap(), +/// TfportName { kind: TfportKind::Rear, port: 0, link: 0, vlan: None }, +/// ); +/// ``` +/// +/// # Errors +/// +/// Returns an error if `name` lacks the `tfport` prefix, has an unrecognized +/// device kind, or has malformed port/link/vlan fields. +pub fn parse_tfport_name(name: &str) -> Result { + let body = name.strip_prefix(TFPORT_DEVICE_PREFIX).ok_or_else(|| { + format!("{name} missing expected prefix {TFPORT_DEVICE_PREFIX}") + })?; + + // The device kind is the leading alphabetic run (`qsfp`, `rear`), the + // remainder carries the port/link/vlan numbers. + let split = body + .find(|c: char| !c.is_ascii_alphabetic()) + .ok_or_else(|| format!("{name} has no port id"))?; + let (kind_str, rest) = body.split_at(split); + let kind = TfportKind::from_token(kind_str).ok_or_else(|| { + format!("{name} has unsupported device kind {kind_str}") + })?; + + let (port_link, vlan_str) = match rest.split_once('.') { + Some((port_link, vlan)) => (port_link, Some(vlan)), + None => (rest, None), + }; + + let (port, link) = port_link + .split_once('_') + .ok_or_else(|| format!("{name} has no link id"))?; + + let port = port + .parse::() + .map_err(|_| format!("{name} has invalid port {port}"))?; + + let link = link + .parse::() + .map_err(|_| format!("{name} has invalid link id {link}"))?; + + let vlan = match vlan_str { + None => None, + // A second `.` (e.g. `tfportqsfp10_0.100.200`) leaves a non-numeric + // remainder, so the parse below rejects it. + Some(vlan) => Some( + vlan.parse::() + .map_err(|_| format!("{name} has invalid vlan {vlan}"))?, + ), + }; + + Ok(TfportName { + kind, + port, + link, + vlan, + }) +} + +/// Build a dpd [`PortId`] from a parsed tfport kind and port number. +/// +/// # Errors +/// +/// Returns an error if the synthesized port name is not a valid dpd `qsfp` or +/// `rear` port identifier. +/// +/// [`PortId`]: dpd_client::types::PortId +pub fn tfport_port_id( + kind: TfportKind, + port: u8, +) -> Result { + use dpd_client::types; + + let port_name = format!("{}{port}", kind.token()); + match kind { + TfportKind::Qsfp => types::Qsfp::try_from(&port_name) + .map(types::PortId::Qsfp) + .map_err(|e| format!("bad qsfp port name {port_name}: {e}")), + TfportKind::Rear => types::Rear::try_from(&port_name) + .map(types::PortId::Rear) + .map_err(|e| format!("bad rear port name {port_name}: {e}")), + } +} + +/// Resolve a tfport datalink name (e.g. `tfportrear0_0`) to the dpd +/// `(PortId, LinkId)` pair that names the switch port and link. +/// +/// # Errors +/// +/// Returns an error if `ifname` is not a valid tfport datalink name or its +/// kind and port do not form a valid dpd port identifier. +/// +/// [`PortId`]: dpd_client::types::PortId +/// [`LinkId`]: dpd_client::types::LinkId +pub fn port_link_from_ifname( + ifname: &str, +) -> Result<(dpd_client::types::PortId, dpd_client::types::LinkId), String> { + let tfport = parse_tfport_name(ifname)?; + let port_id = tfport_port_id(tfport.kind, tfport.port)?; + // Breakout lanes surface as distinct datalinks (`qsfp0_0`, `qsfp0_1`, ...), + // sharing one `PortId`. The parsed link is the lane, which is the dpd + // `LinkId`. + let link_id = dpd_client::types::LinkId(tfport.link); + Ok((port_id, link_id)) +} + +#[cfg(test)] +mod tests { + use super::*; + use proptest::prelude::*; + + fn name( + kind: TfportKind, + port: u8, + link: u8, + vlan: Option, + ) -> TfportName { + TfportName { + kind, + port, + link, + vlan, + } + } + + #[test] + fn test_tfport_parser() { + // Valid qsfp (front-panel) names. + assert_eq!( + parse_tfport_name("tfportqsfp10_0").unwrap(), + name(TfportKind::Qsfp, 10, 0, None) + ); + assert_eq!( + parse_tfport_name("tfportqsfp10_0.100").unwrap(), + name(TfportKind::Qsfp, 10, 0, Some(100)) + ); + assert_eq!( + parse_tfport_name("tfportqsfp1_1").unwrap(), + name(TfportKind::Qsfp, 1, 1, None) + ); + + // Valid rear (backplane) names. + assert_eq!( + parse_tfport_name("tfportrear0_0").unwrap(), + name(TfportKind::Rear, 0, 0, None) + ); + assert_eq!( + parse_tfport_name("tfportrear31_0.200").unwrap(), + name(TfportKind::Rear, 31, 0, Some(200)) + ); + + // Malformed names. + assert!(parse_tfport_name("fportqsfp10_0").is_err()); + assert!(parse_tfport_name("10_0").is_err()); + assert!(parse_tfport_name("tfportqsfp10").is_err()); + assert!(parse_tfport_name("tfportqsfp_10").is_err()); + assert!(parse_tfport_name("tfportqsfp0_").is_err()); + assert!(parse_tfport_name("tfportqsfp10_10_10").is_err()); + assert!(parse_tfport_name("tfportqsfp10.100_0").is_err()); + + // Unsupported or missing device kind. + assert!(parse_tfport_name("tfportfoo0_0").is_err()); + assert!(parse_tfport_name("tfport0_0").is_err()); + + // Invalid numeric components. + assert!(parse_tfport_name("tfportqsfp1X_0.100").is_err()); + assert!(parse_tfport_name("tfportqsfp10_X.100").is_err()); + assert!(parse_tfport_name("tfportqsfp10_0.X").is_err()); + } + + proptest! { + /// Any well-formed name round-trips: formatting a `kind`, `port`, + /// `link`, `vlan` tuple and parsing it back yields the same components. + /// The parser is purely syntactic, so the full u8/u16 ranges are + /// exercised. + #[test] + fn prop_roundtrip( + is_rear in any::(), + port in any::(), + link in any::(), + vlan in proptest::option::of(any::()), + ) { + let kind = if is_rear { TfportKind::Rear } else { TfportKind::Qsfp }; + let mut ifname = format!("tfport{}{port}_{link}", kind.token()); + if let Some(vlan) = vlan { + ifname.push_str(&format!(".{vlan}")); + } + prop_assert_eq!( + parse_tfport_name(&ifname).unwrap(), + TfportName { kind, port, link, vlan }, + ); + } + + /// Parsing arbitrary input never panics; it always returns a `Result`. + #[test] + fn prop_never_panics(ifname in ".*") { + let _ = parse_tfport_name(&ifname); + } + } +} diff --git a/mg-lower/src/ddm.rs b/mg-lower/src/ddm.rs index 8e1744851..efa9e6d90 100644 --- a/mg-lower/src/ddm.rs +++ b/mg-lower/src/ddm.rs @@ -5,10 +5,13 @@ use crate::log::ddm_log; #[cfg(target_os = "illumos")] use ddm_admin_client::Client; -use ddm_api_types_versions::latest::net::TunnelOrigin; +use ddm_api_types_versions::latest::net::{MulticastOrigin, TunnelOrigin}; use oxnet::Ipv6Net; use slog::Logger; -use std::{net::Ipv6Addr, sync::Arc}; +use std::{ + net::{Ipv6Addr, SocketAddr}, + sync::Arc, +}; use crate::platform::Ddm; @@ -107,7 +110,71 @@ pub(crate) fn remove_tunnel_routes<'a, I: Iterator>( } } +/// Create a new DDM admin client. +/// +/// In production the lower half runs in the same zone as DDM, so `addr` is +/// `None` and the client targets the default `localhost:8000`. Tests pass an +/// explicit `addr` to reach a DDM listening elsewhere (for example a +/// dynamically assigned port in an integration harness). #[cfg(target_os = "illumos")] -pub fn new_ddm_client(log: &Logger) -> Client { - Client::new("http://localhost:8000", log.clone()) +pub fn new_ddm_client(log: &Logger, addr: Option) -> Client { + let host = match addr { + Some(addr) => format!("http://{addr}"), + None => "http://localhost:8000".to_string(), + }; + Client::new(&host, log.clone()) +} + +pub(crate) fn add_multicast_routes< + 'a, + I: Iterator, +>( + client: &impl Ddm, + routes: I, + rt: &Arc, + log: &Logger, +) { + let routes: Vec = routes.cloned().collect(); + if routes.is_empty() { + return; + } + let resp = + rt.block_on(async { client.advertise_multicast_groups(&routes).await }); + if let Err(e) = resp { + ddm_log!(log, + error, + "advertise multicast groups error: {e}"; + "error" => format!("{e}"), + "groups" => format!("{routes:#?}") + ); + } +} + +pub(crate) fn remove_multicast_routes< + 'a, + I: Iterator, +>( + client: &impl Ddm, + routes: I, + rt: &Arc, + log: &Logger, +) { + let routes: Vec = routes.cloned().collect(); + if routes.is_empty() { + return; + } + let resp = + rt.block_on(async { client.withdraw_multicast_groups(&routes).await }); + match resp { + Err(e) => ddm_log!(log, + error, + "withdraw multicast groups error: {e}"; + "groups" => format!("{routes:#?}") + ), + Ok(_) => ddm_log!(log, + debug, + "withdrew multicast groups"; + "groups" => format!("{routes:#?}") + ), + } } diff --git a/mg-lower/src/dendrite.rs b/mg-lower/src/dendrite.rs index 1de75f1da..d934f65fb 100644 --- a/mg-lower/src/dendrite.rs +++ b/mg-lower/src/dendrite.rs @@ -16,12 +16,11 @@ use slog::Logger; use std::{ collections::{BTreeSet, HashSet}, hash::Hash, - net::{IpAddr, Ipv4Addr, Ipv6Addr}, + net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}, sync::Arc, time::Duration, }; -const TFPORT_QSFP_DEVICE_PREFIX: &str = "tfportqsfp"; const UNIT_DPD: &str = "dpd"; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] @@ -329,68 +328,12 @@ where Ok(()) } -// Translate a tfport name into the underlying (port, link, vlan) tuple. -// tfportqsfp10_0 would translate to (10, 0, None) -// tfportqsfp10_0.100 would translate to (10, 0, Some(100)) -// TODO this is gross, use link type properties rather than futzing -// around with strings. -fn parse_tfport_name(name: &str) -> Result<(u8, u8, Option), Error> { - let body = - name.strip_prefix(TFPORT_QSFP_DEVICE_PREFIX) - .ok_or(Error::Tfport(format!( - "{} missing expected prefix {}", - name, TFPORT_QSFP_DEVICE_PREFIX - )))?; - let fields: Vec<&str> = body.split('.').collect(); - let (port, link) = fields[0] - .split_once('_') - .ok_or(Error::Tfport(format!("{} has no link id", name)))?; - - let port = port.parse::().map_err(|_| { - Error::Tfport(format!("{} has invalid port {}", name, port)) - })?; - - let link = link.parse::().map_err(|_| { - Error::Tfport(format!("{} has invalid link id {}", name, link)) - })?; - - let vlan = match fields.len() { - 1 => Ok(None), - 2 => fields[1].parse::().map(Some).map_err(|_| { - Error::Tfport(format!("{} has invalid vlan {}", name, fields[1])) - }), - _ => Err(Error::Tfport(format!( - "{} has multiple vlan deliminators", - name - ))), - }?; - - Ok((port, link, vlan)) -} - -#[test] -fn test_tfport_parser() { - // Test valid names - assert_eq!(parse_tfport_name("tfportqsfp10_0").unwrap(), (10, 0, None)); - assert_eq!( - parse_tfport_name("tfportqsfp10_0.100").unwrap(), - (10, 0, Some(100)) - ); - assert_eq!(parse_tfport_name("tfportqsfp1_1").unwrap(), (1, 1, None)); - - // test malformed names - assert!(parse_tfport_name("fportqsfp10_0").is_err()); - assert!(parse_tfport_name("10_0").is_err()); - assert!(parse_tfport_name("tfportqsfp10").is_err()); - assert!(parse_tfport_name("tfportqsfp_10").is_err()); - assert!(parse_tfport_name("tfportqsfp0_").is_err()); - assert!(parse_tfport_name("tfportqsfp10_10_10").is_err()); - assert!(parse_tfport_name("tfportqsfp10.100_0").is_err()); - - // test invalid components - assert!(parse_tfport_name("tfportqsfp1X_0.100").is_err()); - assert!(parse_tfport_name("tfportqsfp10_X.100").is_err()); - assert!(parse_tfport_name("tfportqsfp10_0.X").is_err()); +/// Resolve a tfport datalink name (e.g. `tfportrear0_0`) to the dpd +/// `(PortId, LinkId)` pair that names the switch port and link. +pub(crate) fn port_link_from_ifname( + ifname: &str, +) -> Result<(types::PortId, types::LinkId), Error> { + mg_common::tfport::port_link_from_ifname(ifname).map_err(Error::Tfport) } fn get_port_and_link( @@ -402,18 +345,7 @@ fn get_port_and_link( && nh6.is_unicast_link_local() && let Some(ref iface) = path.nexthop_interface { - let (port, link, _vlan) = parse_tfport_name(iface)?; - let port_name = format!("qsfp{port}"); - let port_id = types::Qsfp::try_from(&port_name) - .map(types::PortId::Qsfp) - .map_err(|e| { - Error::Tfport(format!( - "bad port name ifname: {iface} port name: {port_name}: {e}", - )) - })?; - // TODO breakout considerations - let link_id = types::LinkId(link); - return Ok((port_id, link_id)); + return port_link_from_ifname(iface); } // Standard nexthop resolution for numbered peers @@ -437,19 +369,7 @@ fn resolve_port_and_link( } }; - let (port, link, _vlan) = parse_tfport_name(&ifname)?; - let port_name = format!("qsfp{port}"); - let port_id = types::Qsfp::try_from(&port_name) - .map(types::PortId::Qsfp) - .map_err(|e| { - Error::Tfport(format!( - "bad port name ifname: {ifname} port name: {port_name}: {e}" - )) - })?; - - // TODO breakout considerations - let link_id = types::LinkId(link); - Ok((port_id, link_id)) + port_link_from_ifname(&ifname) } pub(crate) fn get_routes_for_prefix( @@ -560,16 +480,21 @@ pub(crate) fn get_routes_for_prefix( Ok(result.into_iter().collect()) } -/// Create a new Dendrite/dpd client. The lower half always runs on the same -/// host/zone as the underlying platform. +/// Create a new Dendrite (DPD) client. +/// +/// On the rack, the lower half runs in the same zone as DPD, so `addr` is +/// `None` and the client targets `localhost` on the default DPD port. Tests +/// pass an explicit `addr` to reach a DPD listening elsewhere (for example a +/// dynamically assigned port in an integration harness). #[cfg(target_os = "illumos")] -pub fn new_dpd_client(log: &Logger) -> DpdClient { +pub fn new_dpd_client(log: &Logger, addr: Option) -> DpdClient { let client_state = dpd_client::ClientState { tag: MG_LOWER_TAG.into(), log: log.clone(), }; - DpdClient::new( - &format!("http://localhost:{}", dpd_client::default_port()), - client_state, - ) + let host = match addr { + Some(addr) => format!("http://{addr}"), + None => format!("http://localhost:{}", dpd_client::default_port()), + }; + DpdClient::new(&host, client_state) } diff --git a/mg-lower/src/lib.rs b/mg-lower/src/lib.rs index 33096b5e1..042b4db16 100644 --- a/mg-lower/src/lib.rs +++ b/mg-lower/src/lib.rs @@ -40,6 +40,7 @@ mod ddm; mod dendrite; mod error; mod log; +pub mod mrib; mod platform; #[cfg(test)] diff --git a/mg-lower/src/mrib.rs b/mg-lower/src/mrib.rs new file mode 100644 index 000000000..0ecce505e --- /dev/null +++ b/mg-lower/src/mrib.rs @@ -0,0 +1,498 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! MRIB (Multicast Routing Information Base) lower-half synchronization. +//! +//! Advertises locally originated MRIB multicast groups to the DDM admin API, +//! which distributes them across the underlay to other sleds and racks. This is +//! the multicast analog of the unicast lower-half's tunnel-endpoint origination. +//! +//! Origination reads the local MRIB (`loc_mrib`) and is watch-driven, with the +//! periodic resync covering only missed notifications, the same shape as the +//! unicast lower-half's crate-level loop [`crate::run`]. +//! +//! The inbound membership half (resolving DDM-imported routes to switch +//! replication members in DPD) lives in ddmd (`ddm::mcast`), where both inputs, +//! the imported set and the peer table, are owned in-process. +//! +//! ## Data Flow +//! +//! ```text +//! Origination (MRIB -> DDM -> underlay) +//! MRIB (loc_mrib changes) +//! | [MribChangeNotification] +//! v [MulticastOrigin] (diff vs DDM originated) +//! DDM admin API --[DDM exchange]--> other sleds/racks +//! ``` +//! +//! See RFD 488 for the multicast architecture. + +use crate::ddm::{add_multicast_routes, remove_multicast_routes}; +use crate::platform::Ddm; +use ddm_api_types_versions::latest::net::{MulticastOrigin, OverlayMulticast}; +use rdb::Mrib; +use rdb::types::{MribChangeNotification, MulticastAddr, MulticastRoute}; +use slog::{Logger, debug, error, info}; +use std::collections::HashSet; +use std::sync::Arc; +use std::sync::mpsc::{RecvTimeoutError, channel}; +use std::thread::sleep; +use std::time::Duration; + +pub(crate) const MG_LOWER_MRIB_TAG: &str = "mg-lower-mrib"; + +/// Interval between periodic MRIB full syncs. +/// +/// Covers the case where an MRIB change notification is missed. Mirrors the +/// unicast lower-half's 1s resync cadence in `crate::run`. +const MRIB_PERIODIC_SYNC_INTERVAL: Duration = Duration::from_secs(1); + +/// Convert an MRIB `MulticastRoute` to a DDM `MulticastOrigin`. +fn ddm_origin(route: &MulticastRoute) -> MulticastOrigin { + MulticastOrigin { + // The MRIB key's group is a `MulticastAddr`, multicast by construction, + // so promotion to the validated overlay type cannot fail here. + overlay_group: OverlayMulticast::new(route.key.group().ip()) + .expect("MRIB group is multicast by construction"), + underlay_group: route.underlay_group, + vni: route.key.vni(), + metric: 0, + source: route.key.source(), + } +} + +/// Run the MRIB origination loop. +/// +/// This function loops forever, watching for MRIB changes and advertising +/// locally originated multicast groups to DDM. +/// +/// It runs on the calling thread, so callers are responsible for running it in +/// a separate thread if asynchronous execution is required. +pub fn run( + mrib: Mrib, + log: Logger, + rt: Arc, + ddm: &impl Ddm, +) { + loop { + let (tx, rx) = channel(); + + // Register as MRIB watcher + mrib.watch(MG_LOWER_MRIB_TAG.into(), tx); + + // Initial full sync + if let Err(e) = full_sync(&mrib, ddm, &log, &rt) { + error!(log, "MRIB full sync failed: {e}"); + info!(log, "restarting MRIB sync loop in one second"); + // Drop this iteration's watcher before retrying. The continue + // re-registers a fresh watcher, so without this the failed + // registration would accumulate in the MRIB watcher list on every + // retry. + mrib.unwatch(MG_LOWER_MRIB_TAG); + // Pause before retrying to keep a persistent failure from spinning. + // This is a backoff floor, kept independent of the resync cadence so + // tuning one does not silently change the other. + // + // Note: the unicast lower-half pauses in the same way + // (see `crate::run`). + sleep(Duration::from_secs(1)); + continue; + } + + // Handle incremental changes + loop { + match rx.recv_timeout(MRIB_PERIODIC_SYNC_INTERVAL) { + Ok(notification) => { + if let Err(e) = + handle_change(&mrib, notification, ddm, &log, &rt) + { + error!(log, "MRIB change handling failed: {e}"); + } + } + // if we've not received updates in the timeout interval, do a + // full sync in case something has changed out from under us. + Err(RecvTimeoutError::Timeout) => { + if let Err(e) = full_sync(&mrib, ddm, &log, &rt) { + error!(log, "MRIB periodic sync failed: {e}"); + info!(log, "restarting MRIB sync loop in one second"); + // Same backoff floor as the initial-sync failure, so a + // persistent failure retries at that pace rather than + // every resync interval. The unicast lower-half pauses + // in the same way (see `crate::run`). + sleep(Duration::from_secs(1)); + } + } + Err(RecvTimeoutError::Disconnected) => { + error!(log, "MRIB watcher disconnected"); + break; + } + } + } + } +} + +/// Perform a full synchronization of MRIB origination to DDM. +/// +/// Compares the current MRIB `loc_mrib` with what DDM has advertised and +/// reconciles any differences. +pub(crate) fn full_sync( + mrib: &Mrib, + ddm: &D, + log: &Logger, + rt: &Arc, +) -> Result<(), String> { + // Get current MRIB state (installed/selected routes) + let mrib_routes = mrib.loc_mrib(); + + // Convert to DDM MulticastOrigin set + let mrib_origins: HashSet = + mrib_routes.values().map(ddm_origin).collect(); + + // Get current DDM advertised state + let ddm_current: HashSet = rt + .block_on(async { ddm.get_originated_multicast_groups().await }) + .map_err(|e| format!("failed to get DDM multicast groups: {e}"))? + .into_inner() + .into_iter() + .collect(); + + // Compute diff + let to_add: Vec<_> = mrib_origins.difference(&ddm_current).collect(); + let to_remove: Vec<_> = ddm_current.difference(&mrib_origins).collect(); + + if !to_add.is_empty() { + info!( + log, + "MRIB sync: adding {} multicast groups to DDM", + to_add.len() + ); + add_multicast_routes(ddm, to_add.into_iter(), rt, log); + } + + if !to_remove.is_empty() { + info!( + log, + "MRIB sync: removing {} multicast groups from DDM", + to_remove.len() + ); + remove_multicast_routes(ddm, to_remove.into_iter(), rt, log); + } + + Ok(()) +} + +/// Handle an incremental MRIB change notification. +fn handle_change( + mrib: &Mrib, + notification: MribChangeNotification, + ddm: &D, + log: &Logger, + rt: &Arc, +) -> Result<(), String> { + // Get current DDM state for comparison + let ddm_current: HashSet = rt + .block_on(async { ddm.get_originated_multicast_groups().await }) + .map_err(|e| format!("failed to get DDM multicast groups: {e}"))? + .into_inner() + .into_iter() + .collect(); + + let mut to_add = Vec::new(); + let mut to_remove = Vec::new(); + + for key in notification.changed { + // Check if route exists in `loc_mrib` (installed) + if let Some(route) = mrib.get_selected_route(&key) { + let origin = ddm_origin(&route); + if !ddm_current.contains(&origin) { + to_add.push(origin); + } + } else { + // Route is not in `loc_mrib`, so we need to find matching DDM + // origin. We check all DDM origins to find any that match this key + for ddm_origin in &ddm_current { + // Reconstruct the key from the DDM origin to compare + if let Ok(overlay_group) = + MulticastAddr::try_from(ddm_origin.overlay_group.ip()) + && let Ok(ddm_key) = rdb::types::MulticastRouteKey::new( + ddm_origin.source, + overlay_group, + ddm_origin.vni, + ) + && ddm_key == key + { + to_remove.push(ddm_origin.clone()); + } + } + } + } + + if !to_add.is_empty() { + debug!(log, "MRIB change: adding {} multicast groups", to_add.len()); + add_multicast_routes(ddm, to_add.iter(), rt, log); + } + + if !to_remove.is_empty() { + debug!( + log, + "MRIB change: removing {} multicast groups", + to_remove.len() + ); + remove_multicast_routes(ddm, to_remove.iter(), rt, log); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::platform::test::TestDdm; + use rdb::test::get_test_db; + use rdb::types::{ + MulticastRouteKey, MulticastSourceProtocol, UnderlayMulticastIpv6, + }; + use std::net::Ipv6Addr; + + fn discard_logger() -> Logger { + Logger::root(slog::Discard, slog::o!()) + } + + /// Runtime handle for `full_sync`'s internal `block_on`. Tests run on a + /// plain thread (not a tokio worker), so blocking on this handle is safe. + fn runtime() -> (tokio::runtime::Runtime, Arc) { + let rt = tokio::runtime::Runtime::new().expect("build runtime"); + let handle = Arc::new(rt.handle().clone()); + (rt, handle) + } + + fn test_underlay() -> UnderlayMulticastIpv6 { + UnderlayMulticastIpv6::new(Ipv6Addr::new(0xff04, 0, 0, 0, 0, 0, 0, 1)) + .expect("valid underlay address") + } + + /// Build an any-source (*,G) static route for the given IPv4 group. + fn asm_route(a: u8, b: u8, c: u8, d: u8) -> MulticastRoute { + let group = MulticastAddr::new_v4(a, b, c, d).expect("valid group"); + let key = MulticastRouteKey::any_source(group); + MulticastRoute::new( + key, + test_underlay(), + MulticastSourceProtocol::Static, + ) + } + + /// Build an any-source (*,G) static route for an IPv6 group in + /// `ff0e::/16`, with a distinct underlay so the 1:1 overlay-to-underlay + /// mapping holds alongside `asm_route` entries. + fn asm_route_v6(last: u16) -> MulticastRoute { + let group = MulticastAddr::new_v6([0xff0e, 0, 0, 0, 0, 0, 0, last]) + .expect("valid group"); + let underlay = UnderlayMulticastIpv6::new(Ipv6Addr::new( + 0xff04, 0, 0, 0, 0, 0, 0x1, last, + )) + .expect("valid underlay address"); + let key = MulticastRouteKey::any_source(group); + MulticastRoute::new(key, underlay, MulticastSourceProtocol::Static) + } + + #[test] + fn full_sync_advertises_groups_missing_from_ddm() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_full_sync_add", log.clone()).expect("db"); + let route = asm_route(225, 1, 1, 1); + db.add_static_mcast_routes(std::slice::from_ref(&route)) + .expect("add route"); + + let ddm = TestDdm::default(); + full_sync(db.mrib(), &ddm, &log, &handle).expect("full sync"); + + let originated = ddm.multicast_originated.lock().unwrap(); + assert_eq!(originated.len(), 1); + assert_eq!(originated[0], ddm_origin(&route)); + } + + #[test] + fn full_sync_advertises_v4_and_v6_overlay_groups() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_full_sync_v6", log.clone()).expect("db"); + let v4 = asm_route(225, 1, 1, 1); + let v6 = asm_route_v6(1); + db.add_static_mcast_routes(&[v4.clone(), v6.clone()]) + .expect("add routes"); + + let ddm = TestDdm::default(); + full_sync(db.mrib(), &ddm, &log, &handle).expect("full sync"); + + let originated = ddm.multicast_originated.lock().unwrap(); + assert_eq!(originated.len(), 2); + assert!(originated.contains(&ddm_origin(&v4))); + assert!(originated.contains(&ddm_origin(&v6))); + } + + #[test] + fn full_sync_withdraws_groups_absent_from_mrib() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_full_sync_remove", log.clone()).expect("db"); + + let stale = asm_route(225, 9, 9, 9); + let ddm = TestDdm::default(); + ddm.multicast_originated + .lock() + .unwrap() + .push(ddm_origin(&stale)); + + full_sync(db.mrib(), &ddm, &log, &handle).expect("full sync"); + + assert!(ddm.multicast_originated.lock().unwrap().is_empty()); + } + + #[test] + fn full_sync_in_sync_makes_no_changes() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_full_sync_noop", log.clone()).expect("db"); + let route = asm_route(225, 5, 5, 5); + db.add_static_mcast_routes(std::slice::from_ref(&route)) + .expect("add route"); + + let ddm = TestDdm::default(); + ddm.multicast_originated + .lock() + .unwrap() + .push(ddm_origin(&route)); + + full_sync(db.mrib(), &ddm, &log, &handle).expect("full sync"); + + let originated = ddm.multicast_originated.lock().unwrap(); + assert_eq!(originated.len(), 1, "in-sync group must not be re-added"); + assert_eq!(originated[0], ddm_origin(&route)); + } + + #[test] + fn full_sync_adds_and_withdraws_together() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_full_sync_mixed", log.clone()).expect("db"); + let keep = asm_route(225, 1, 1, 1); + db.add_static_mcast_routes(std::slice::from_ref(&keep)) + .expect("add route"); + + let ddm = TestDdm::default(); + let stale = asm_route(225, 2, 2, 2); + ddm.multicast_originated + .lock() + .unwrap() + .push(ddm_origin(&stale)); + + full_sync(db.mrib(), &ddm, &log, &handle).expect("full sync"); + + let originated = ddm.multicast_originated.lock().unwrap(); + assert_eq!(originated.len(), 1); + assert_eq!(originated[0], ddm_origin(&keep)); + } + + #[test] + fn handle_change_advertises_newly_installed_group() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_change_add", log.clone()).expect("db"); + let route = asm_route(225, 1, 1, 1); + db.add_static_mcast_routes(std::slice::from_ref(&route)) + .expect("add route"); + + let ddm = TestDdm::default(); + let notification = MribChangeNotification::from(route.key); + handle_change(db.mrib(), notification, &ddm, &log, &handle) + .expect("handle change"); + + let originated = ddm.multicast_originated.lock().unwrap(); + assert_eq!(originated.len(), 1); + assert_eq!(originated[0], ddm_origin(&route)); + } + + #[test] + fn handle_change_withdraws_removed_group() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_change_remove", log.clone()).expect("db"); + + // The group is advertised by DDM but never installed in the MRIB, + // modeling a route that was removed from `loc_mrib`. + let removed = asm_route(225, 9, 9, 9); + let ddm = TestDdm::default(); + ddm.multicast_originated + .lock() + .unwrap() + .push(ddm_origin(&removed)); + + let notification = MribChangeNotification::from(removed.key); + handle_change(db.mrib(), notification, &ddm, &log, &handle) + .expect("handle change"); + + assert!(ddm.multicast_originated.lock().unwrap().is_empty()); + } + + #[test] + fn handle_change_is_noop_when_already_advertised() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_change_noop", log.clone()).expect("db"); + let route = asm_route(225, 5, 5, 5); + db.add_static_mcast_routes(std::slice::from_ref(&route)) + .expect("add route"); + + let ddm = TestDdm::default(); + ddm.multicast_originated + .lock() + .unwrap() + .push(ddm_origin(&route)); + + let notification = MribChangeNotification::from(route.key); + handle_change(db.mrib(), notification, &ddm, &log, &handle) + .expect("handle change"); + + let originated = ddm.multicast_originated.lock().unwrap(); + assert_eq!( + originated.len(), + 1, + "already-advertised group must not be re-added" + ); + assert_eq!(originated[0], ddm_origin(&route)); + } + + #[test] + fn handle_change_withdraws_only_the_matching_group() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_change_selective", log.clone()).expect("db"); + + // DDM advertises two groups: only one is named in the change set and is + // absent from the MRIB, which means only that one must be withdrawn. + // This exercises the key-reconstruction match in the removal path. + let removed = asm_route(225, 1, 1, 1); + let other = asm_route(225, 2, 2, 2); + let ddm = TestDdm::default(); + { + let mut originated = ddm.multicast_originated.lock().unwrap(); + originated.push(ddm_origin(&removed)); + originated.push(ddm_origin(&other)); + } + + let notification = MribChangeNotification::from(removed.key); + handle_change(db.mrib(), notification, &ddm, &log, &handle) + .expect("handle change"); + + let originated = ddm.multicast_originated.lock().unwrap(); + assert_eq!( + originated.len(), + 1, + "unrelated group must remain advertised" + ); + assert_eq!(originated[0], ddm_origin(&other)); + } +} diff --git a/mg-lower/src/platform.rs b/mg-lower/src/platform.rs index 8350f245b..5115b457c 100644 --- a/mg-lower/src/platform.rs +++ b/mg-lower/src/platform.rs @@ -1,4 +1,4 @@ -//! This crate contains traits that decouple mg-lower from the underlying +//! This module contains traits that decouple mg-lower from the underlying //! platform. This is useful for testing mg-lower while not having to //! have a running dpd, ddmd, or switch zone. //! @@ -8,7 +8,7 @@ use std::net::IpAddr; use std::time::Duration; use ddm_admin_client::types::Error as DdmError; -use ddm_api_types_versions::latest::net::TunnelOrigin; +use ddm_api_types_versions::latest::net::{MulticastOrigin, TunnelOrigin}; use dpd_client::types::{Error as DpdError, *}; // Use the crate-specific re-exports of progenitor_client types to avoid // version conflicts: dpd_client re-exports from progenitor-client 0.11 (via @@ -217,6 +217,46 @@ pub trait Ddm { ddm_admin_client::ResponseValue<()>, ddm_admin_client::Error, >; + + /// Get multicast group subscriptions originated by this router. + /// + /// Each `MulticastOrigin` pairs an overlay group address with its + /// underlay mapping (ff04::/64) and optional source for (S,G) routes. + /// + /// Method names follow the DDM admin API convention + /// (`originated_multicast_groups`, not `originated_multicast_origins`). + async fn get_originated_multicast_groups( + &self, + ) -> Result< + ddm_admin_client::ResponseValue>, + ddm_admin_client::Error, + >; + + /// Advertise multicast group subscriptions to DDM peers. + /// + /// Each entry is a `MulticastOrigin` pairing an overlay group + /// with its ff04::/64 underlay mapping. + #[allow(clippy::ptr_arg)] + async fn advertise_multicast_groups<'a>( + &'a self, + body: &'a Vec, + ) -> Result< + ddm_admin_client::ResponseValue<()>, + ddm_admin_client::Error, + >; + + /// Withdraw multicast group subscriptions from DDM peers. + /// + /// Each entry is a `MulticastOrigin` pairing an overlay group + /// with its ff04::/64 underlay mapping. + #[allow(clippy::ptr_arg)] + async fn withdraw_multicast_groups<'a>( + &'a self, + body: &'a Vec, + ) -> Result< + ddm_admin_client::ResponseValue<()>, + ddm_admin_client::Error, + >; } /// This trait wraps the methods that have expectations about switch zone @@ -353,6 +393,7 @@ impl Dpd for ProductionDpd { /// Production ddm trait that simply passes through calls to a ddm client. #[cfg(target_os = "illumos")] +#[derive(Clone)] pub struct ProductionDdm { pub client: DdmClient, } @@ -406,6 +447,35 @@ impl Ddm for ProductionDdm { > { self.client.withdraw_tunnel_endpoints(body).await } + + async fn get_originated_multicast_groups( + &self, + ) -> Result< + ddm_admin_client::ResponseValue>, + ddm_admin_client::Error, + > { + self.client.get_originated_multicast_groups().await + } + + async fn advertise_multicast_groups<'a>( + &'a self, + body: &'a Vec, + ) -> Result< + ddm_admin_client::ResponseValue<()>, + ddm_admin_client::Error, + > { + self.client.advertise_multicast_groups(body).await + } + + async fn withdraw_multicast_groups<'a>( + &'a self, + body: &'a Vec, + ) -> Result< + ddm_admin_client::ResponseValue<()>, + ddm_admin_client::Error, + > { + self.client.withdraw_multicast_groups(body).await + } } /// Production switch zone that uses libnet for route lookups (illumos only). @@ -431,6 +501,7 @@ pub(crate) mod test { use crate::MG_LOWER_TAG; use super::*; + use mg_common::lock; use std::sync::Mutex; use std::{collections::HashMap, net::IpAddr}; @@ -485,7 +556,7 @@ pub(crate) mod test { link_id: &LinkId, ) -> Result, DpdClientError> { - let links = self.links.lock().unwrap(); + let links = lock!(self.links); let link = links .iter() .find(|x| &x.port_id == port_id && &x.link_id == link_id); @@ -503,10 +574,7 @@ pub(crate) mod test { dpd_client::ResponseValue>, DpdClientError, > { - let result = self - .v4_routes - .lock() - .unwrap() + let result = lock!(self.v4_routes) .get(cidr) .cloned() .unwrap_or(Vec::default()); @@ -520,10 +588,7 @@ pub(crate) mod test { dpd_client::ResponseValue>, DpdClientError, > { - let result = self - .v6_routes - .lock() - .unwrap() + let result = lock!(self.v6_routes) .get(cidr) .cloned() .unwrap_or(Vec::default()); @@ -535,7 +600,7 @@ pub(crate) mod test { addr: &Ipv6Entry, ) -> Result, DpdClientError> { - self.loopback.lock().unwrap().replace(addr.clone()); + lock!(self.loopback).replace(addr.clone()); Ok(dpd_response_ok!(())) } @@ -546,7 +611,7 @@ pub(crate) mod test { dpd_client::ResponseValue>, DpdClientError, > { - let links = self.links.lock().unwrap(); + let links = lock!(self.links); let result = links .iter() .filter(|x| match filter { @@ -613,7 +678,7 @@ pub(crate) mod test { RouteTarget::V4(v4) => Route::V4(v4.clone()), RouteTarget::V6(v6) => Route::V6(v6.clone()), }; - let mut routes = self.v4_routes.lock().unwrap(); + let mut routes = lock!(self.v4_routes); match routes.get_mut(&body.cidr) { Some(targets) => { targets.push(route); @@ -630,7 +695,7 @@ pub(crate) mod test { body: &'a Ipv6RouteUpdate, ) -> Result, DpdClientError> { - let mut routes = self.v6_routes.lock().unwrap(); + let mut routes = lock!(self.v6_routes); match routes.get_mut(&body.cidr) { Some(targets) => { targets.push(body.target.clone()); @@ -650,7 +715,7 @@ pub(crate) mod test { tgt_ip: &'a IpAddr, ) -> Result, DpdClientError> { - let mut routes = self.v4_routes.lock().unwrap(); + let mut routes = lock!(self.v4_routes); if let Some(targets) = routes.get_mut(cidr) { targets.retain(|x| match (x, tgt_ip) { (Route::V4(x), IpAddr::V4(ip)) => { @@ -678,7 +743,7 @@ pub(crate) mod test { tgt_ip: &'a std::net::Ipv6Addr, ) -> Result, DpdClientError> { - let mut routes = self.v6_routes.lock().unwrap(); + let mut routes = lock!(self.v6_routes); if let Some(targets) = routes.get_mut(cidr) { targets.retain(|x| { !(x.tgt_ip == *tgt_ip @@ -700,6 +765,7 @@ pub(crate) mod test { pub(crate) struct TestDdm { pub(crate) tunnel_originated: Mutex>, pub(crate) originated: Mutex>, + pub(crate) multicast_originated: Mutex>, } impl Default for TestDdm { @@ -707,6 +773,7 @@ pub(crate) mod test { Self { tunnel_originated: Mutex::new(Vec::default()), originated: Mutex::new(Vec::default()), + multicast_originated: Mutex::new(Vec::default()), } } } @@ -718,9 +785,7 @@ pub(crate) mod test { ddm_admin_client::ResponseValue>, ddm_admin_client::Error, > { - Ok(ddm_response_ok!( - self.tunnel_originated.lock().unwrap().clone() - )) + Ok(ddm_response_ok!(lock!(self.tunnel_originated).clone())) } async fn get_originated( @@ -729,7 +794,7 @@ pub(crate) mod test { ddm_admin_client::ResponseValue>, ddm_admin_client::Error, > { - Ok(ddm_response_ok!(self.originated.lock().unwrap().clone())) + Ok(ddm_response_ok!(lock!(self.originated).clone())) } async fn advertise_prefixes<'a>( @@ -739,7 +804,7 @@ pub(crate) mod test { ddm_admin_client::ResponseValue<()>, ddm_admin_client::Error, > { - self.originated.lock().unwrap().extend(body); + lock!(self.originated).extend(body); Ok(ddm_response_ok!(())) } @@ -750,7 +815,7 @@ pub(crate) mod test { ddm_admin_client::ResponseValue<()>, ddm_admin_client::Error, > { - self.tunnel_originated.lock().unwrap().extend(body.clone()); + lock!(self.tunnel_originated).extend(body.clone()); Ok(ddm_response_ok!(())) } @@ -761,10 +826,38 @@ pub(crate) mod test { ddm_admin_client::ResponseValue<()>, ddm_admin_client::Error, > { - self.tunnel_originated - .lock() - .unwrap() - .retain(|x| !body.contains(x)); + lock!(self.tunnel_originated).retain(|x| !body.contains(x)); + Ok(ddm_response_ok!(())) + } + + async fn get_originated_multicast_groups( + &self, + ) -> Result< + ddm_admin_client::ResponseValue>, + ddm_admin_client::Error, + > { + Ok(ddm_response_ok!(lock!(self.multicast_originated).clone())) + } + + async fn advertise_multicast_groups<'a>( + &'a self, + body: &'a Vec, + ) -> Result< + ddm_admin_client::ResponseValue<()>, + ddm_admin_client::Error, + > { + lock!(self.multicast_originated).extend(body.clone()); + Ok(ddm_response_ok!(())) + } + + async fn withdraw_multicast_groups<'a>( + &'a self, + body: &'a Vec, + ) -> Result< + ddm_admin_client::ResponseValue<()>, + ddm_admin_client::Error, + > { + lock!(self.multicast_originated).retain(|x| !body.contains(x)); Ok(ddm_response_ok!(())) } } diff --git a/mgd/src/main.rs b/mgd/src/main.rs index 571291c9d..39eec937b 100644 --- a/mgd/src/main.rs +++ b/mgd/src/main.rs @@ -114,6 +114,20 @@ struct RunArgs { /// SocketAddr for the BGP Dispatcher to listen on. #[arg(long, default_value = "[::]:179")] bgp_dispatcher_addr: SocketAddr, + + /// SocketAddr the Dendrite (DPD) API is listening on. When unset, the lower + /// half targets DPD at its default port on localhost (the co-located switch + /// zone). Set this to point the lower half at a DPD elsewhere, such as a + /// dynamically assigned port in an integration harness. + #[arg(long)] + dendrite_addr: Option, + + /// SocketAddr the DDM admin API is listening on. When unset, the lower half + /// targets DDM at its default localhost address (the co-located switch + /// zone). Set this to point the lower half at a DDM elsewhere, such as a + /// dynamically assigned port in an integration harness. + #[arg(long)] + ddm_addr: Option, } fn main() { @@ -167,24 +181,44 @@ async fn run(args: RunArgs) { #[cfg(all(feature = "mg-lower", target_os = "illumos"))] { - let rt = Arc::new(tokio::runtime::Handle::current()); - let ctx = context.clone(); - let log = log.clone(); - let db = ctx.db.clone(); - let stats = context.mg_lower_stats.clone(); let dpd = mg_lower::ProductionDpd { - client: mg_lower::new_dpd_client(&log), + client: mg_lower::new_dpd_client(&log, args.dendrite_addr), }; let ddm = mg_lower::ProductionDdm { - client: mg_lower::new_ddm_client(&log), + client: mg_lower::new_ddm_client(&log, args.ddm_addr), }; - let sw = mg_lower::ProductionSwitchZone {}; - Builder::new() - .name("mg-lower".to_string()) - .spawn(move || { - mg_lower::run(ctx.tep, db, log, stats, rt, &dpd, &ddm, &sw); - }) - .expect("failed to start mg-lower"); + + // Unicast lower-half: sync the unicast RIB to Dendrite and + // advertise tunnel endpoint routes to DDM. + { + let rt = Arc::new(tokio::runtime::Handle::current()); + let ctx = context.clone(); + let log = log.clone(); + let db = ctx.db.clone(); + let stats = context.mg_lower_stats.clone(); + let ddm = ddm.clone(); + let sw = mg_lower::ProductionSwitchZone {}; + Builder::new() + .name("mg-lower".to_string()) + .spawn(move || { + mg_lower::run(ctx.tep, db, log, stats, rt, &dpd, &ddm, &sw); + }) + .expect("failed to start mg-lower"); + } + + // Multicast lower-half: advertise locally originated MRIB groups to + // DDM. Underlay replication membership is reconciled in `ddmd`. + { + let rt = Arc::new(tokio::runtime::Handle::current()); + let log = log.clone(); + let mrib = context.db.mrib().clone(); + Builder::new() + .name("mg-lower-mrib".to_string()) + .spawn(move || { + mg_lower::mrib::run(mrib, log, rt, &ddm); + }) + .expect("failed to start mg-lower mrib sync"); + } } start_bgp_routers( diff --git a/openapi/ddm-admin/ddm-admin-2.0.0-45d40c.json.gitstub b/openapi/ddm-admin/ddm-admin-2.0.0-45d40c.json.gitstub new file mode 100644 index 000000000..b5ab91ce1 --- /dev/null +++ b/openapi/ddm-admin/ddm-admin-2.0.0-45d40c.json.gitstub @@ -0,0 +1 @@ +561931c41eafde867f931229e799d1b30df7a1d4:openapi/ddm-admin/ddm-admin-2.0.0-45d40c.json diff --git a/openapi/ddm-admin/ddm-admin-2.0.0-45d40c.json b/openapi/ddm-admin/ddm-admin-3.0.0-974538.json similarity index 68% rename from openapi/ddm-admin/ddm-admin-2.0.0-45d40c.json rename to openapi/ddm-admin/ddm-admin-3.0.0-974538.json index e891ca0e6..89d25b8db 100644 --- a/openapi/ddm-admin/ddm-admin-2.0.0-45d40c.json +++ b/openapi/ddm-admin/ddm-admin-3.0.0-974538.json @@ -6,7 +6,7 @@ "url": "https://oxide.computer", "email": "api@oxide.computer" }, - "version": "2.0.0" + "version": "3.0.0" }, "paths": { "/disable-stats": { @@ -51,6 +51,94 @@ } } }, + "/multicast_group": { + "put": { + "operationId": "advertise_multicast_groups", + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Set_of_MulticastOrigin", + "type": "array", + "items": { + "$ref": "#/components/schemas/MulticastOrigin" + }, + "uniqueItems": true + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "operationId": "withdraw_multicast_groups", + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Set_of_MulticastOrigin", + "type": "array", + "items": { + "$ref": "#/components/schemas/MulticastOrigin" + }, + "uniqueItems": true + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/multicast_groups": { + "get": { + "operationId": "get_multicast_groups", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Set_of_MulticastRoute", + "type": "array", + "items": { + "$ref": "#/components/schemas/MulticastRoute" + }, + "uniqueItems": true + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/originated": { "get": { "operationId": "get_originated", @@ -79,6 +167,34 @@ } } }, + "/originated_multicast_groups": { + "get": { + "operationId": "get_originated_multicast_groups", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Set_of_MulticastOrigin", + "type": "array", + "items": { + "$ref": "#/components/schemas/MulticastOrigin" + }, + "uniqueItems": true + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/originated_tunnel_endpoints": { "get": { "operationId": "get_originated_tunnel_endpoints", @@ -463,6 +579,104 @@ "type": "string", "pattern": "^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))\\/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8])$" }, + "MulticastOrigin": { + "description": "Origin information for a multicast group announcement.\n\nAnalogous to `TunnelOrigin` but for multicast groups. Represents a subscription to a multicast group that should be advertised via DDM. `overlay_group` is the application-visible multicast address (e.g., 233.252.0.1 or ff0e::1), while `underlay_group` is the mapped admin-local scoped IPv6 address (ff04::X) used in the underlay network.", + "type": "object", + "properties": { + "metric": { + "description": "Metric for path selection (lower is better).\n\nUsed for multi-rack replication optimization. Excluded from identity (Hash/Eq) so that metric changes update an existing entry rather than creating a duplicate.", + "default": 0, + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "overlay_group": { + "description": "The overlay multicast group address (IPv4 or IPv6). This is the group address visible to applications. Validated at construction to be a multicast address.", + "type": "string", + "format": "ip" + }, + "source": { + "nullable": true, + "description": "Optional source address for Source-Specific Multicast (S,G) routes. `None` for Any-Source Multicast (*,G) routes.", + "default": null, + "type": "string", + "format": "ip" + }, + "underlay_group": { + "description": "The underlay multicast group address (ff04::X). Validated at construction to be within ff04::/64.", + "type": "string", + "format": "ipv6" + }, + "vni": { + "description": "VNI for this multicast group (identifies the VPC/network context).", + "default": 77, + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + "required": [ + "overlay_group", + "underlay_group" + ] + }, + "MulticastPathHop": { + "description": "A single hop in the multicast path, carrying metadata for replication optimization (RFD 488).", + "type": "object", + "properties": { + "downstream_subscriber_count": { + "description": "Number of downstream subscribers reachable via this hop.", + "default": 0, + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "router_id": { + "description": "Router identifier (hostname).", + "type": "string" + }, + "underlay_addr": { + "description": "The underlay address of this router (for replication targeting).", + "type": "string", + "format": "ipv6" + } + }, + "required": [ + "router_id", + "underlay_addr" + ] + }, + "MulticastRoute": { + "description": "A multicast route learned via DDM.\n\nCarries a `MulticastOrigin` (overlay group + ff04::/64 underlay mapping) and the path vector from the originating subscriber through intermediate transit routers.", + "type": "object", + "properties": { + "nexthop": { + "description": "Underlay nexthop address (DDM peer that advertised this route). Used to associate the route with a peer for expiration.", + "type": "string", + "format": "ipv6" + }, + "origin": { + "description": "The multicast group origin information.", + "allOf": [ + { + "$ref": "#/components/schemas/MulticastOrigin" + } + ] + }, + "path": { + "description": "Path vector from the originating subscriber outward. Each hop records the router that redistributed this subscription announcement. Used for loop detection on pull and for future replication optimization in multi-rack topologies.", + "default": [], + "type": "array", + "items": { + "$ref": "#/components/schemas/MulticastPathHop" + } + } + }, + "required": [ + "nexthop", + "origin" + ] + }, "PathVector": { "type": "object", "properties": { @@ -482,7 +696,7 @@ ] }, "PeerInfo": { - "description": "Information about a DDM peer.", + "description": "Peer information with an optional interface name.", "type": "object", "properties": { "addr": { @@ -492,6 +706,12 @@ "host": { "type": "string" }, + "if_name": { + "nullable": true, + "description": "Interface name the peer was discovered on (e.g., \"tfportrear0_0\").", + "default": null, + "type": "string" + }, "kind": { "$ref": "#/components/schemas/RouterKind" }, diff --git a/openapi/ddm-admin/ddm-admin-latest.json b/openapi/ddm-admin/ddm-admin-latest.json index 4eb6e8dbc..ca00e9746 120000 --- a/openapi/ddm-admin/ddm-admin-latest.json +++ b/openapi/ddm-admin/ddm-admin-latest.json @@ -1 +1 @@ -ddm-admin-2.0.0-45d40c.json \ No newline at end of file +ddm-admin-3.0.0-974538.json \ No newline at end of file diff --git a/smf/ddm/manifest.xml b/smf/ddm/manifest.xml index 5e5da1cf2..f9c7f7425 100644 --- a/smf/ddm/manifest.xml +++ b/smf/ddm/manifest.xml @@ -19,7 +19,7 @@ - + diff --git a/tests/src/ddm.rs b/tests/src/ddm.rs index 988ed973d..c5b13404f 100644 --- a/tests/src/ddm.rs +++ b/tests/src/ddm.rs @@ -5,7 +5,9 @@ use anyhow::{Result, anyhow}; use client_common::{eprintln_nopipe, println_nopipe}; use ddm_admin_client::Client; -use ddm_api_types_versions::latest::net::TunnelOrigin; +use ddm_api_types_versions::latest::net::{ + MulticastOrigin, TunnelOrigin, UnderlayMulticastIpv6, Vni, +}; use slog::{Drain, Logger}; use std::env; use std::net::Ipv6Addr; @@ -190,6 +192,32 @@ impl<'a> RouterZone<'a> { self.zone.zexec("pkill ddmd") } + /// Wait for an SMF service in this zone to come online, failing fast if + /// it lands in maintenance. A service that silently fails here otherwise + /// surfaces much later as an opaque peering assertion failure. + fn wait_for_service_online(&self, fmri: &str) -> Result<()> { + for _ in 0..30 { + let state = self + .zone + .zexec(&format!("svcs -Ho state {fmri}")) + .unwrap_or_default(); + match state.trim() { + "online" => return Ok(()), + "maintenance" => { + return Err(anyhow!( + "{fmri} entered maintenance in zone {}", + self.zone.name, + )); + } + _ => sleep(Duration::from_secs(1)), + } + } + Err(anyhow!( + "timed out waiting for {fmri} to come online in zone {}", + self.zone.name, + )) + } + fn start_router(&self, restart_dpd: bool) -> Result<()> { let addrs = self.ifx[1..] .iter() @@ -224,9 +252,8 @@ impl<'a> RouterZone<'a> { ))?; self.zone.zexec("svcadm refresh dendrite:default")?; self.zone.zexec("svcadm enable dendrite:default")?; - // wait for dendrite to come up - println_nopipe!("wait 10s for dendrite to come up ..."); - sleep(Duration::from_secs(10)); + println_nopipe!("waiting for dendrite to come online ..."); + self.wait_for_service_online("dendrite:default")?; self.zone.zexec( "svccfg -s tfport setprop config/pkt_source = none", )?; @@ -235,6 +262,8 @@ impl<'a> RouterZone<'a> { )?; self.zone.zexec("svcadm refresh tfport:default")?; self.zone.zexec("svcadm enable tfport")?; + println_nopipe!("waiting for tfport to come online ..."); + self.wait_for_service_online("tfport:default")?; } self.zone.zexec(&format!( "{} {ddm} --kind transit --dendrite {} {} &> /opt/ddmd.log &", @@ -648,6 +677,69 @@ async fn run_trio_tests( println_nopipe!("tunnel endpoint withdraw passed"); + // Multicast group advertise/withdraw across the trio. Mirrors how + // mg-lower in the switch zone publishes overlay→underlay multicast + // bindings: the transit router originates an advertisement, and the + // server routers learn it via DDM exchange. + wait_for_eq!(multicast_originated_count(&t1).await?, 0); + + // One origin per overlay address family: the overlay group is opaque to + // ddm (any multicast IpAddr), but both families must survive the + // advertise/exchange/withdraw round trip. + let mcast_origin_v4 = MulticastOrigin { + overlay_group: "233.252.0.1".parse().unwrap(), + underlay_group: UnderlayMulticastIpv6::new( + "ff04::100".parse().unwrap(), + ) + .unwrap(), + vni: Vni::try_from(77u32).unwrap(), + source: None, + metric: 0, + }; + let mcast_origin_v6 = MulticastOrigin { + overlay_group: "ff0e::1".parse().unwrap(), + underlay_group: UnderlayMulticastIpv6::new( + "ff04::101".parse().unwrap(), + ) + .unwrap(), + vni: Vni::try_from(77u32).unwrap(), + source: None, + metric: 0, + }; + + t1.advertise_multicast_groups(&vec![ + mcast_origin_v4.clone(), + mcast_origin_v6.clone(), + ]) + .await?; + + wait_for_eq!(multicast_originated_count(&t1).await?, 2); + wait_for_eq!(multicast_group_count(&t1).await?, 0); + wait_for_eq!(multicast_group_count(&s1).await?, 2); + wait_for_eq!(multicast_group_count(&s2).await?, 2); + + println_nopipe!("multicast group advertise passed"); + + // Server router restart: s1's view of the multicast group must + // converge again after ddmd restarts. wait_for_eq tolerates the + // restart window via unwrap_or sentinel. + zs1.stop_router()?; + zs1.start_router(false)?; + let s1 = Client::new("http://10.0.0.1:8000", log.clone()); + wait_for_eq!(multicast_group_count(&s1).await.unwrap_or(99), 2); + + println_nopipe!("multicast router restart passed"); + + t1.withdraw_multicast_groups(&vec![mcast_origin_v4, mcast_origin_v6]) + .await?; + + wait_for_eq!(multicast_originated_count(&t1).await?, 0); + wait_for_eq!(multicast_group_count(&t1).await?, 0); + wait_for_eq!(multicast_group_count(&s1).await?, 0); + wait_for_eq!(multicast_group_count(&s2).await?, 0); + + println_nopipe!("multicast group withdraw passed"); + Ok(()) } @@ -818,6 +910,14 @@ async fn tunnel_originated_endpoint_count(c: &Client) -> Result { Ok(c.get_originated_tunnel_endpoints().await?.len()) } +async fn multicast_group_count(c: &Client) -> Result { + Ok(c.get_multicast_groups().await?.len()) +} + +async fn multicast_originated_count(c: &Client) -> Result { + Ok(c.get_originated_multicast_groups().await?.len()) +} + fn init_logger() -> Logger { let decorator = slog_term::TermDecorator::new().build(); let drain = slog_term::FullFormat::new(decorator).build().fuse();