From 6d959dec3d149055683845233f52c705e8562a08 Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Wed, 1 Apr 2026 08:13:40 +0000 Subject: [PATCH 01/16] [multicast] DDM multicast exchange: V4 protocol, MRIB sync, M2P hooks Adds multicast group subscription distribution to the DDM exchange protocol with a V4 version bump (frozen V3 types for wire compat). Key changes: - V4 exchange protocol with multicast support (V3 peers are unaffected) - UnderlayMulticastIpv6 validated newtype moved to mg-common (ff04::/64) (moved from rdb types) - MRIB->DDM sync in mg-lower/mrib.rs - OPTE M2P hooks for learned multicast routes (requires OPTE #924) - Atomic update_imported_mcast on Db (single lock for import/delete/diff, which is a bit different from the tunnel work) - Collapsed send_update dispatch - Shared pull handler helpers (collect_underlay_tunnel, collect_multicast) - MulticastPathHop constructor - Some serde round-trip and validation tests, including for version handling Stacked on zl/mrib (MRIB: Multicast RIB implementation [#675](https://github.com/oxidecomputer/maghemite/pull/675)). --- Cargo.lock | 2 + ddm-admin-client/src/lib.rs | 22 + ddm-api/src/lib.rs | 41 +- ddm-types/versions/src/latest.rs | 3 + ddm-types/versions/src/lib.rs | 2 + .../versions/src/multicast_support/db.rs | 62 ++ .../src/multicast_support/exchange.rs | 76 +++ .../versions/src/multicast_support/mod.rs | 9 + ddm/src/admin.rs | 69 ++- ddm/src/db.rs | 137 ++++- ddm/src/discovery.rs | 9 +- ddm/src/exchange.rs | 579 +++++++++++++++--- ddm/src/sm.rs | 209 ++++++- ddm/src/sys.rs | 95 ++- ddmadm/src/main.rs | 123 ++++ mg-common/Cargo.toml | 4 + mg-common/src/net.rs | 222 ++++++- mg-lower/src/ddm.rs | 56 +- mg-lower/src/lib.rs | 1 + mg-lower/src/mrib.rs | 234 +++++++ mg-lower/src/platform.rs | 95 +++ .../ddm-admin-1.0.0-b6eac7.json.gitstub | 1 + ...6eac7.json => ddm-admin-2.0.0-3dc476.json} | 216 ++++++- openapi/ddm-admin/ddm-admin-latest.json | 2 +- 24 files changed, 2167 insertions(+), 102 deletions(-) create mode 100644 ddm-types/versions/src/multicast_support/db.rs create mode 100644 ddm-types/versions/src/multicast_support/exchange.rs create mode 100644 ddm-types/versions/src/multicast_support/mod.rs create mode 100644 mg-lower/src/mrib.rs create mode 100644 openapi/ddm-admin/ddm-admin-1.0.0-b6eac7.json.gitstub rename openapi/ddm-admin/{ddm-admin-1.0.0-b6eac7.json => ddm-admin-2.0.0-3dc476.json} (64%) diff --git a/Cargo.lock b/Cargo.lock index e3f50d301..f8c9e9e89 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3792,11 +3792,13 @@ dependencies = [ "clap", "libc", "libnet", + "omicron-common", "oximeter", "oximeter-producer", "oxnet", "schemars 0.8.22", "serde", + "serde_json", "slog", "slog-async", "slog-bunyan", diff --git a/ddm-admin-client/src/lib.rs b/ddm-admin-client/src/lib.rs index ea45cea2d..0d22065fe 100644 --- a/ddm-admin-client/src/lib.rs +++ b/ddm-admin-client/src/lib.rs @@ -39,3 +39,25 @@ impl std::hash::Hash for types::TunnelOrigin { self.metric.hash(state); } } + +impl std::cmp::PartialEq for types::MulticastOrigin { + fn eq(&self, other: &Self) -> bool { + self.overlay_group.eq(&other.overlay_group) + && self.underlay_group.eq(&other.underlay_group) + && self.vni.eq(&other.vni) + && self.source.eq(&other.source) + } +} + +impl std::cmp::Eq for types::MulticastOrigin {} + +/// Metric is excluded from identity so that metric changes update +/// an existing entry rather than creating a duplicate. +impl std::hash::Hash for types::MulticastOrigin { + fn hash(&self, state: &mut H) { + self.overlay_group.hash(state); + self.underlay_group.hash(state); + self.vni.hash(state); + self.source.hash(state); + } +} diff --git a/ddm-api/src/lib.rs b/ddm-api/src/lib.rs index 546797dc1..905942b59 100644 --- a/ddm-api/src/lib.rs +++ b/ddm-api/src/lib.rs @@ -10,7 +10,7 @@ use dropshot::Path; use dropshot::RequestContext; use dropshot::TypedBody; use dropshot_api_manager_types::api_versions; -use mg_common::net::TunnelOrigin; +use mg_common::net::{MulticastOrigin, TunnelOrigin}; use oxnet::Ipv6Net; use std::collections::{HashMap, HashSet}; @@ -26,6 +26,7 @@ api_versions!([ // | example for the next person. // v // (next_int, IDENT), + (2, MULTICAST_SUPPORT), (1, INITIAL), ]); @@ -100,6 +101,44 @@ pub trait DdmAdminApi { request: TypedBody>, ) -> Result; + #[endpoint { + method = GET, + path = "/originated_multicast_groups", + versions = VERSION_MULTICAST_SUPPORT.. + }] + async fn get_originated_multicast_groups( + ctx: RequestContext, + ) -> Result>, HttpError>; + + #[endpoint { + method = GET, + path = "/multicast_groups", + versions = VERSION_MULTICAST_SUPPORT.. + }] + async fn get_multicast_groups( + ctx: RequestContext, + ) -> Result>, HttpError>; + + #[endpoint { + method = PUT, + path = "/multicast_group", + versions = VERSION_MULTICAST_SUPPORT.. + }] + async fn advertise_multicast_groups( + ctx: RequestContext, + request: TypedBody>, + ) -> Result; + + #[endpoint { + method = DELETE, + path = "/multicast_group", + versions = VERSION_MULTICAST_SUPPORT.. + }] + async fn withdraw_multicast_groups( + ctx: RequestContext, + request: TypedBody>, + ) -> Result; + #[endpoint { method = PUT, path = "/sync" }] async fn sync( ctx: RequestContext, diff --git a/ddm-types/versions/src/latest.rs b/ddm-types/versions/src/latest.rs index 080576012..c2d29e3e0 100644 --- a/ddm-types/versions/src/latest.rs +++ b/ddm-types/versions/src/latest.rs @@ -15,9 +15,12 @@ pub mod db { pub use crate::v1::db::PeerStatus; pub use crate::v1::db::RouterKind; pub use crate::v1::db::TunnelRoute; + pub use crate::v2::db::MulticastRoute; } pub mod exchange { pub use crate::v1::exchange::PathVector; pub use crate::v1::exchange::PathVectorV2; + pub use crate::v2::exchange::MulticastPathHop; + pub use crate::v2::exchange::MulticastPathVector; } diff --git a/ddm-types/versions/src/lib.rs b/ddm-types/versions/src/lib.rs index 9f8dcdc71..f723aaf04 100644 --- a/ddm-types/versions/src/lib.rs +++ b/ddm-types/versions/src/lib.rs @@ -32,3 +32,5 @@ pub mod latest; #[path = "initial/mod.rs"] pub mod v1; +#[path = "multicast_support/mod.rs"] +pub mod v2; diff --git a/ddm-types/versions/src/multicast_support/db.rs b/ddm-types/versions/src/multicast_support/db.rs new file mode 100644 index 000000000..f6cdf4d01 --- /dev/null +++ b/ddm-types/versions/src/multicast_support/db.rs @@ -0,0 +1,62 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::net::Ipv6Addr; + +use mg_common::net::MulticastOrigin; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::v2::exchange::MulticastPathHop; + +/// A multicast route learned via DDM. +/// +/// Carries both the group origin and the path vector from the +/// originating subscriber through intermediate transit routers. +/// The path enables loop detection and (in multi-rack topologies) +/// replication optimizations per [RFD 488] in the future. +/// +/// Equality and hashing consider only `origin` and `nexthop` so that +/// a route update with a longer path replaces the existing entry in +/// hash-based collections. +/// +/// [RFD 488]: https://rfd.shared.oxide.computer/rfd/0488 +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct MulticastRoute { + /// The multicast group origin information. + pub origin: MulticastOrigin, + + /// Underlay nexthop address (DDM peer that advertised this route). + /// Used to associate the route with a peer for expiration. + pub nexthop: Ipv6Addr, + + /// Path vector from the originating subscriber outward. + /// Each hop records the router that redistributed this + /// subscription announcement. Used for loop detection on pull + /// and for future replication optimization in multi-rack + /// topologies. + #[serde(default)] + pub path: Vec, +} + +impl PartialEq for MulticastRoute { + fn eq(&self, other: &Self) -> bool { + self.origin == other.origin && self.nexthop == other.nexthop + } +} + +impl Eq for MulticastRoute {} + +impl std::hash::Hash for MulticastRoute { + fn hash(&self, state: &mut H) { + self.origin.hash(state); + self.nexthop.hash(state); + } +} + +impl From for MulticastOrigin { + fn from(x: MulticastRoute) -> Self { + x.origin + } +} diff --git a/ddm-types/versions/src/multicast_support/exchange.rs b/ddm-types/versions/src/multicast_support/exchange.rs new file mode 100644 index 000000000..4aa528ed5 --- /dev/null +++ b/ddm-types/versions/src/multicast_support/exchange.rs @@ -0,0 +1,76 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::net::Ipv6Addr; + +/// A single hop in the multicast path, carrying metadata needed for +/// replication optimization. +/// +/// Unlike unicast paths which only need hostnames, multicast hops carry +/// additional information for computing optimal replication points per +/// [RFD 488]. +/// +/// [RFD 488]: https://rfd.shared.oxide.computer/rfd/0488 +#[derive( + Debug, Clone, PartialEq, Eq, Hash, Deserialize, Serialize, JsonSchema, +)] +pub struct MulticastPathHop { + /// Router identifier (hostname). + pub router_id: String, + + /// The underlay address of this router (for replication targeting). + pub underlay_addr: Ipv6Addr, + + /// Number of downstream subscribers reachable via this hop. + /// Used for load-aware replication decisions in multi-rack + /// topologies. + #[serde(default)] + pub downstream_subscriber_count: u32, +} + +impl MulticastPathHop { + /// Create a hop with the given router identity and a zero subscriber + /// count. The count will be populated once transit routers track + /// downstream subscriber counts for load-aware replication (RFD 488). + pub fn new(router_id: String, underlay_addr: Ipv6Addr) -> Self { + Self { + router_id, + underlay_addr, + downstream_subscriber_count: 0, + } + } +} + +/// Multicast group subscription announcement propagating through DDM. +/// +/// The path records the sequence of routers from the original subscriber +/// toward the current receiving router. Currently, this is used for loop +/// detection: if our router_id appears in the path, the announcement has +/// already traversed us and is dropped. The path structure also carries +/// topology information for future replication optimizations (RFD 488). +#[derive( + Debug, Clone, PartialEq, Eq, Hash, Deserialize, Serialize, JsonSchema, +)] +pub struct MulticastPathVector { + /// The multicast group origin information. + pub origin: mg_common::net::MulticastOrigin, + + /// The path from the original subscriber to the current router. + /// Ordered from subscriber outward (subscriber router first). + pub path: Vec, +} + +impl MulticastPathVector { + /// Append a hop to this path vector. + pub fn with_hop(&self, hop: MulticastPathHop) -> Self { + let mut path = self.path.clone(); + path.push(hop); + Self { + origin: self.origin.clone(), + path, + } + } +} diff --git a/ddm-types/versions/src/multicast_support/mod.rs b/ddm-types/versions/src/multicast_support/mod.rs new file mode 100644 index 000000000..066113e24 --- /dev/null +++ b/ddm-types/versions/src/multicast_support/mod.rs @@ -0,0 +1,9 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Types from API version 2 (MULTICAST_SUPPORT) that add multicast +//! group management to the DDM admin API. + +pub mod db; +pub mod exchange; diff --git a/ddm/src/admin.rs b/ddm/src/admin.rs index 6d49a368b..51a50677a 100644 --- a/ddm/src/admin.rs +++ b/ddm/src/admin.rs @@ -7,7 +7,7 @@ use crate::sm::{AdminEvent, Event, PrefixSet, SmContext}; use ddm_api::DdmAdminApi; use ddm_api::ddm_admin_api_mod; use ddm_types::admin::{EnableStatsRequest, ExpirePathParams, PrefixMap}; -use ddm_types::db::{PeerInfo, TunnelRoute}; +use ddm_types::db::{MulticastRoute, PeerInfo, TunnelRoute}; use ddm_types::exchange::PathVector; use dropshot::ApiDescription; use dropshot::ApiDescriptionBuildErrors; @@ -21,7 +21,7 @@ use dropshot::Path; use dropshot::RequestContext; use dropshot::TypedBody; use mg_common::lock; -use mg_common::net::TunnelOrigin; +use mg_common::net::{MulticastOrigin, TunnelOrigin}; use oxnet::Ipv6Net; use slog::{Logger, error, info}; use std::collections::{HashMap, HashSet}; @@ -333,6 +333,71 @@ impl DdmAdminApi for DdmAdminApiImpl { Ok(HttpResponseUpdatedNoContent()) } + async fn get_originated_multicast_groups( + ctx: RequestContext, + ) -> Result>, HttpError> { + let ctx = lock!(ctx.context()); + let originated = ctx + .db + .originated_mcast() + .map_err(|e| HttpError::for_internal_error(e.to_string()))?; + Ok(HttpResponseOk(originated)) + } + + async fn get_multicast_groups( + ctx: RequestContext, + ) -> Result>, HttpError> { + let ctx = lock!(ctx.context()); + let imported = ctx.db.imported_mcast(); + Ok(HttpResponseOk(imported)) + } + + async fn advertise_multicast_groups( + ctx: RequestContext, + request: TypedBody>, + ) -> Result { + let ctx = lock!(ctx.context()); + let groups = request.into_inner(); + slog::info!(ctx.log, "advertise multicast groups: {groups:#?}"); + ctx.db + .originate_mcast(&groups) + .map_err(|e| HttpError::for_internal_error(e.to_string()))?; + + for e in &ctx.event_channels { + e.send(Event::Admin(AdminEvent::Announce(PrefixSet::Multicast( + groups.clone(), + )))) + .map_err(|e| { + HttpError::for_internal_error(format!("admin event send: {e}")) + })?; + } + + Ok(HttpResponseUpdatedNoContent()) + } + + async fn withdraw_multicast_groups( + ctx: RequestContext, + request: TypedBody>, + ) -> Result { + let ctx = lock!(ctx.context()); + let groups = request.into_inner(); + slog::info!(ctx.log, "withdraw multicast groups: {groups:#?}"); + ctx.db + .withdraw_mcast(&groups) + .map_err(|e| HttpError::for_internal_error(e.to_string()))?; + + for e in &ctx.event_channels { + e.send(Event::Admin(AdminEvent::Withdraw(PrefixSet::Multicast( + groups.clone(), + )))) + .map_err(|e| { + HttpError::for_internal_error(format!("admin event send: {e}")) + })?; + } + + Ok(HttpResponseUpdatedNoContent()) + } + async fn sync( ctx: RequestContext, ) -> Result { diff --git a/ddm/src/db.rs b/ddm/src/db.rs index 13338cc43..30eec234a 100644 --- a/ddm/src/db.rs +++ b/ddm/src/db.rs @@ -2,9 +2,9 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use ddm_types::db::{PeerInfo, TunnelRoute}; +use ddm_types::db::{MulticastRoute, PeerInfo, TunnelRoute}; use mg_common::lock; -use mg_common::net::TunnelOrigin; +use mg_common::net::{MulticastOrigin, TunnelOrigin}; use oxnet::{IpNet, Ipv6Net}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -21,6 +21,10 @@ const ORIGINATE: &str = "originate"; /// tunnel endpoints. const TUNNEL_ORIGINATE: &str = "tunnel_originate"; +/// The handle used to open a persistent key-value tree for originated +/// multicast groups. +const MCAST_ORIGINATE: &str = "mcast_originate"; + #[derive(thiserror::Error, Debug)] pub enum Error { #[error("datastore error {0}")] @@ -48,6 +52,7 @@ pub struct DbData { pub peers: HashMap, pub imported: HashSet, pub imported_tunnel: HashSet, + pub imported_mcast: HashSet, } unsafe impl Sync for Db {} @@ -85,6 +90,14 @@ impl Db { lock!(self.data).imported_tunnel.len() } + pub fn imported_mcast(&self) -> HashSet { + lock!(self.data).imported_mcast.clone() + } + + pub fn imported_mcast_count(&self) -> usize { + lock!(self.data).imported_mcast.len() + } + pub fn import(&self, r: &HashSet) { lock!(self.data).imported.extend(r.clone()); } @@ -93,6 +106,10 @@ impl Db { lock!(self.data).imported_tunnel.extend(r.clone()); } + pub fn import_mcast(&self, r: &HashSet) { + lock!(self.data).imported_mcast.extend(r.clone()); + } + pub fn delete_import(&self, r: &HashSet) { let imported = &mut lock!(self.data).imported; for x in r { @@ -107,6 +124,38 @@ impl Db { } } + pub fn delete_import_mcast(&self, r: &HashSet) { + let imported = &mut lock!(self.data).imported_mcast; + for x in r { + imported.remove(x); + } + } + + /// Atomically import and delete multicast routes under a single lock, + /// returning the effective difference (additions + removals) against the + /// state before mutation. + /// + /// This avoids a TOCTOU race where concurrent mutations between separate + /// lock acquisitions could produce an incorrect view difference. + pub fn update_imported_mcast( + &self, + import: &HashSet, + remove: &HashSet, + ) -> (HashSet, HashSet) { + let mut data = lock!(self.data); + + let before = data.imported_mcast.clone(); + data.imported_mcast.extend(import.iter().cloned()); + + for x in remove { + data.imported_mcast.remove(x); + } + + let to_add = data.imported_mcast.difference(&before).cloned().collect(); + let to_del = before.difference(&data.imported_mcast).cloned().collect(); + (to_add, to_del) + } + pub fn originate(&self, prefixes: &HashSet) -> Result<(), Error> { let tree = self.persistent_data.open_tree(ORIGINATE)?; for p in prefixes { @@ -129,6 +178,19 @@ impl Db { Ok(()) } + pub fn originate_mcast( + &self, + origins: &HashSet, + ) -> Result<(), Error> { + let tree = self.persistent_data.open_tree(MCAST_ORIGINATE)?; + for o in origins { + let entry = serde_json::to_string(o)?; + tree.insert(entry.as_str(), "")?; + } + tree.flush()?; + Ok(()) + } + pub fn originated(&self) -> Result, Error> { let tree = self.persistent_data.open_tree(ORIGINATE)?; let result = tree @@ -178,6 +240,7 @@ impl Db { return None; } }; + let value = String::from_utf8_lossy(&key); let value: TunnelOrigin = match serde_json::from_str(&value) { Ok(item) => item, @@ -199,6 +262,44 @@ impl Db { Ok(self.originated_tunnel()?.len()) } + pub fn originated_mcast(&self) -> Result, Error> { + let tree = self.persistent_data.open_tree(MCAST_ORIGINATE)?; + let result = tree + .scan_prefix(vec![]) + .filter_map(|item| { + let (key, _value) = match item { + Ok(item) => item, + Err(e) => { + error!( + self.log, + "db: error fetching ddm mcast origin entry: {e}" + ); + return None; + } + }; + + let value = String::from_utf8_lossy(&key); + let value: MulticastOrigin = match serde_json::from_str(&value) + { + Ok(item) => item, + Err(e) => { + error!( + self.log, + "db: error parsing ddm mcast origin: {e}" + ); + return None; + } + }; + Some(value) + }) + .collect(); + Ok(result) + } + + pub fn originated_mcast_count(&self) -> Result { + Ok(self.originated_mcast()?.len()) + } + pub fn withdraw(&self, prefixes: &HashSet) -> Result<(), Error> { let tree = self.persistent_data.open_tree(ORIGINATE)?; for p in prefixes { @@ -221,6 +322,19 @@ impl Db { Ok(()) } + pub fn withdraw_mcast( + &self, + origins: &HashSet, + ) -> Result<(), Error> { + let tree = self.persistent_data.open_tree(MCAST_ORIGINATE)?; + for o in origins { + let entry = serde_json::to_string(o)?; + tree.remove(entry.as_str())?; + } + tree.flush()?; + Ok(()) + } + /// Set peer info at the given index. Returns true if peer information was /// changed. pub fn set_peer(&self, index: u32, info: PeerInfo) -> bool { @@ -233,7 +347,11 @@ impl Db { pub fn remove_nexthop_routes( &self, nexthop: Ipv6Addr, - ) -> (HashSet, HashSet) { + ) -> ( + HashSet, + HashSet, + HashSet, + ) { let mut data = lock!(self.data); // Routes are generally held in sets to prevent duplication and provide // handy set-algebra operations. @@ -256,7 +374,18 @@ impl Db { for x in &tnl_removed { data.imported_tunnel.remove(x); } - (removed, tnl_removed) + + let mut mcast_removed = HashSet::new(); + for x in &data.imported_mcast { + if x.nexthop == nexthop { + mcast_removed.insert(x.clone()); + } + } + for x in &mcast_removed { + data.imported_mcast.remove(x); + } + + (removed, tnl_removed, mcast_removed) } pub fn remove_peer(&self, index: u32) { diff --git a/ddm/src/discovery.rs b/ddm/src/discovery.rs index fc4a84e80..dd6da9346 100644 --- a/ddm/src/discovery.rs +++ b/ddm/src/discovery.rs @@ -113,6 +113,7 @@ const ADVERTISE: u8 = 1 << 1; pub enum Version { V2 = 2, V3 = 3, + V4 = 4, } #[derive(Error, Debug)] @@ -136,7 +137,7 @@ pub struct DiscoveryPacket { impl DiscoveryPacket { pub fn new_solicitation(hostname: String, kind: RouterKind) -> Self { Self { - version: Version::V2 as u8, + version: Version::V4 as u8, flags: SOLICIT, hostname, kind, @@ -144,7 +145,7 @@ impl DiscoveryPacket { } pub fn new_advertisement(hostname: String, kind: RouterKind) -> Self { Self { - version: Version::V2 as u8, + version: Version::V4 as u8, flags: ADVERTISE, hostname, kind, @@ -461,12 +462,12 @@ fn handle_advertisement( let version = match version { 2 => Version::V2, 3 => Version::V3, + 4 => Version::V4, x => { err!( ctx.log, ctx.config.if_name, - "unknown protocol version {}, known versions are: 1, 2", - x + "unknown protocol version {x}, known versions are: 2, 3, 4" ); return; } diff --git a/ddm/src/exchange.rs b/ddm/src/exchange.rs index 2c1cc876a..57204cc83 100644 --- a/ddm/src/exchange.rs +++ b/ddm/src/exchange.rs @@ -19,8 +19,10 @@ use crate::db::{Route, effective_route_set}; use crate::discovery::Version; use crate::sm::{Config, Event, PeerEvent, SmContext}; use crate::{dbg, err, inf, wrn}; -use ddm_types::db::{RouterKind, TunnelRoute}; -use ddm_types::exchange::{PathVector, PathVectorV2}; +use ddm_types::db::{MulticastRoute, RouterKind, TunnelRoute}; +use ddm_types::exchange::{ + MulticastPathHop, MulticastPathVector, PathVector, PathVectorV2, +}; use dropshot::ApiDescription; use dropshot::ConfigDropshot; use dropshot::ConfigLogging; @@ -74,10 +76,20 @@ pub struct UpdateV2 { pub tunnel: Option, } +/// THIS TYPE IS FOR DDM PROTOCOL VERSION 3. IT SHALL NEVER CHANGE. THIS TYPE +/// CAN BE REMOVED WHEN DDMV3 CLIENTS AND SERVERS NO LONGER EXIST BUT ITS +/// DEFINITION SHALL NEVER CHANGE. +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] +pub struct UpdateV3 { + pub underlay: Option, + pub tunnel: Option, +} + #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] pub struct Update { pub underlay: Option, pub tunnel: Option, + pub multicast: Option, } impl From for Update { @@ -88,6 +100,7 @@ impl From for Update { announce: value.announce, withdraw: value.withdraw, }), + multicast: None, } } } @@ -97,6 +110,8 @@ impl From for Update { Update { tunnel: value.tunnel.map(TunnelUpdate::from), underlay: value.underlay.map(UnderlayUpdate::from), + // V2 protocol doesn't support multicast + multicast: None, } } } @@ -120,11 +135,31 @@ impl From for UpdateV2 { } } +impl From for Update { + fn from(value: UpdateV3) -> Self { + Update { + underlay: value.underlay, + tunnel: value.tunnel, + multicast: None, + } + } +} + +impl From for UpdateV3 { + fn from(value: Update) -> Self { + UpdateV3 { + underlay: value.underlay, + tunnel: value.tunnel, + } + } +} + impl From for Update { fn from(u: UnderlayUpdate) -> Self { Update { underlay: Some(u), tunnel: None, + multicast: None, } } } @@ -134,6 +169,7 @@ impl From for Update { Update { underlay: None, tunnel: Some(t), + multicast: None, } } } @@ -143,14 +179,35 @@ impl Update { Self { underlay: pr.underlay.map(UnderlayUpdate::announce), tunnel: pr.tunnel.map(TunnelUpdate::announce), + multicast: pr.multicast.map(MulticastUpdate::announce), } } } +impl From for Update { + fn from(m: MulticastUpdate) -> Self { + Update { + underlay: None, + tunnel: None, + multicast: Some(m), + } + } +} + +/// THIS TYPE IS FOR DDM PROTOCOL VERSION 3. IT SHALL NEVER CHANGE. THIS TYPE +/// CAN BE REMOVED WHEN DDMV3 CLIENTS AND SERVERS NO LONGER EXIST BUT ITS +/// DEFINITION SHALL NEVER CHANGE. +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] +pub struct PullResponseV3 { + pub underlay: Option>, + pub tunnel: Option>, +} + #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] pub struct PullResponse { pub underlay: Option>, pub tunnel: Option>, + pub multicast: Option>, } /// THIS TYPE IS FOR DDM PROTOCOL VERSION 2. IT SHALL NEVER CHANGE. THIS TYPE @@ -171,6 +228,18 @@ impl From for PullResponse { tunnel: value .tunnel .map(|x| x.into_iter().map(TunnelOrigin::from).collect()), + // V2 protocol doesn't support multicast + multicast: None, + } + } +} + +impl From for PullResponse { + fn from(value: PullResponseV3) -> Self { + PullResponse { + underlay: value.underlay, + tunnel: value.tunnel, + multicast: None, } } } @@ -180,6 +249,7 @@ impl From> for PullResponse { PullResponse { underlay: Some(value), tunnel: None, + multicast: None, } } } @@ -334,6 +404,47 @@ impl TunnelUpdate { } } +/// Multicast group subscription updates. +/// +/// Carries path-vector information for multicast group subscriptions, +/// enabling loop detection and optimal replication point computation. +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] +pub struct MulticastUpdate { + pub announce: HashSet, + pub withdraw: HashSet, +} + +impl MulticastUpdate { + pub fn announce(groups: HashSet) -> Self { + Self { + announce: groups, + ..Default::default() + } + } + pub fn withdraw(groups: HashSet) -> Self { + Self { + withdraw: groups, + ..Default::default() + } + } + + /// Add a hop to all path vectors in this update. + pub fn with_hop(&self, hop: MulticastPathHop) -> Self { + Self { + announce: self + .announce + .iter() + .map(|pv| pv.with_hop(hop.clone())) + .collect(), + withdraw: self + .withdraw + .iter() + .map(|pv| pv.with_hop(hop.clone())) + .collect(), + } + } +} + #[derive(Error, Debug)] pub enum ExchangeError { #[error("io error: {0}")] @@ -404,15 +515,52 @@ pub(crate) fn withdraw_tunnel( send_update(ctx, config, update.into(), addr, version, rt, log) } -pub(crate) fn do_pull( +pub(crate) fn announce_multicast( + ctx: &SmContext, + config: Config, + groups: HashSet, + addr: Ipv6Addr, + version: Version, + rt: Arc, + log: Logger, +) -> Result<(), ExchangeError> { + let update = MulticastUpdate::announce(groups); + send_update(ctx, config, update.into(), addr, version, rt, log) +} + +pub(crate) fn withdraw_multicast( + ctx: &SmContext, + config: Config, + groups: HashSet, + addr: Ipv6Addr, + version: Version, + rt: Arc, + log: Logger, +) -> Result<(), ExchangeError> { + let update = MulticastUpdate::withdraw(groups); + send_update(ctx, config, update.into(), addr, version, rt, log) +} + +pub(crate) fn do_pull_v4( ctx: &SmContext, addr: &Ipv6Addr, rt: &Arc, ) -> Result { - let uri = format!( - "http://[{}%{}]:{}/v3/pull", - addr, ctx.config.if_index, ctx.config.exchange_port, - ); + let if_index = ctx.config.if_index; + let port = ctx.config.exchange_port; + let uri = format!("http://[{addr}%{if_index}]:{port}/v4/pull"); + let body = do_pull_common(uri, rt)?; + Ok(serde_json::from_slice(&body)?) +} + +pub(crate) fn do_pull_v3( + ctx: &SmContext, + addr: &Ipv6Addr, + rt: &Arc, +) -> Result { + let if_index = ctx.config.if_index; + let port = ctx.config.exchange_port; + let uri = format!("http://[{addr}%{if_index}]:{port}/v3/pull"); let body = do_pull_common(uri, rt)?; Ok(serde_json::from_slice(&body)?) } @@ -464,7 +612,8 @@ pub(crate) fn pull( ) -> Result<(), ExchangeError> { let pr: PullResponse = match version { Version::V2 => do_pull_v2(&ctx, &addr, &rt)?.into(), - Version::V3 => do_pull(&ctx, &addr, &rt)?, + Version::V3 => do_pull_v3(&ctx, &addr, &rt)?.into(), + Version::V4 => do_pull_v4(&ctx, &addr, &rt)?, }; let update = Update::announce(pr); @@ -489,43 +638,14 @@ fn send_update( log: Logger, ) -> Result<(), ExchangeError> { ctx.stats.updates_sent.fetch_add(1, Ordering::Relaxed); - match version { - Version::V2 => { - send_update_v2(ctx, config, update.into(), addr, rt, log) - } - Version::V3 => send_update_v3(ctx, config, update, addr, rt, log), - } -} - -fn send_update_v2( - ctx: &SmContext, - config: Config, - update: UpdateV2, - addr: Ipv6Addr, - rt: Arc, - log: Logger, -) -> Result<(), ExchangeError> { - let payload = serde_json::to_string(&update)?; - let uri = format!( - "http://[{}%{}]:{}/v2/push", - addr, config.if_index, config.exchange_port, - ); - send_update_common(ctx, uri, payload, config, rt, log) -} - -fn send_update_v3( - ctx: &SmContext, - config: Config, - update: Update, - addr: Ipv6Addr, - rt: Arc, - log: Logger, -) -> Result<(), ExchangeError> { - let payload = serde_json::to_string(&update)?; - let uri = format!( - "http://[{}%{}]:{}/v3/push", - addr, config.if_index, config.exchange_port, - ); + let (payload, path) = match version { + Version::V2 => (serde_json::to_string(&UpdateV2::from(update))?, "v2"), + Version::V3 => (serde_json::to_string(&UpdateV3::from(update))?, "v3"), + Version::V4 => (serde_json::to_string(&update)?, "v4"), + }; + let if_index = config.if_index; + let port = config.exchange_port; + let uri = format!("http://[{addr}%{if_index}]:{port}/{path}/push"); send_update_common(ctx, uri, payload, config, rt, log) } @@ -630,9 +750,11 @@ pub fn api_description() -> Result< > { let mut api = ApiDescription::new(); api.register(push_handler_v2)?; - api.register(push_handler)?; + api.register(push_handler_v3)?; + api.register(push_handler_v4)?; api.register(pull_handler_v2)?; - api.register(pull_handler)?; + api.register(pull_handler_v3)?; + api.register(pull_handler_v4)?; Ok(api) } @@ -647,7 +769,16 @@ async fn push_handler_v2( } #[endpoint { method = PUT, path = "/v3/push" }] -async fn push_handler( +async fn push_handler_v3( + ctx: RequestContext>>, + request: TypedBody, +) -> Result { + let update = Update::from(request.into_inner()); + push_handler_common(ctx, update).await +} + +#[endpoint { method = PUT, path = "/v4/push" }] +async fn push_handler_v4( ctx: RequestContext>>, request: TypedBody, ) -> Result { @@ -744,19 +875,15 @@ async fn pull_handler_v2( })) } -#[endpoint { method = GET, path = "/v3/pull" }] -async fn pull_handler( - ctx: RequestContext>>, -) -> Result, HttpError> { - let ctx = ctx.context().lock().await.clone(); - +/// Collect underlay and tunnel routes for pull responses (shared by V3/V4). +fn collect_underlay_tunnel( + ctx: &HandlerContext, +) -> Result<(HashSet, HashSet), HttpError> { let mut underlay = HashSet::new(); let mut tunnel = HashSet::new(); - // Only transit routers redistribute prefixes if ctx.ctx.config.kind == RouterKind::Transit { for route in &ctx.ctx.db.imported() { - // don't redistribute prefixes to their originators if route.nexthop == ctx.peer { continue; } @@ -771,21 +898,20 @@ async fn pull_handler( if route.nexthop == ctx.peer { continue; } - let tv = route.origin; - tunnel.insert(tv); + tunnel.insert(route.origin); } } + let originated = ctx .ctx .db .originated() .map_err(|e| HttpError::for_internal_error(e.to_string()))?; for prefix in &originated { - let pv = PathVector { + underlay.insert(PathVector { destination: *prefix, path: vec![ctx.ctx.hostname.clone()], - }; - underlay.insert(pv); + }); } let originated_tunnel = ctx @@ -794,26 +920,87 @@ async fn pull_handler( .originated_tunnel() .map_err(|e| HttpError::for_internal_error(e.to_string()))?; for prefix in &originated_tunnel { - let tv = TunnelOrigin { + tunnel.insert(TunnelOrigin { overlay_prefix: prefix.overlay_prefix, boundary_addr: prefix.boundary_addr, vni: prefix.vni, metric: prefix.metric, - }; - tunnel.insert(tv); + }); } + Ok((underlay, tunnel)) +} + +/// Collect multicast routes for V4 pull responses. +fn collect_multicast( + ctx: &HandlerContext, +) -> Result, HttpError> { + let mut multicast = HashSet::new(); + + if ctx.ctx.config.kind == RouterKind::Transit { + for route in &ctx.ctx.db.imported_mcast() { + if route.nexthop == ctx.peer { + continue; + } + let hop = MulticastPathHop::new( + ctx.ctx.hostname.clone(), + ctx.ctx.config.addr, + ); + let mut path = route.path.clone(); + path.push(hop); + multicast.insert(MulticastPathVector { + origin: route.origin.clone(), + path, + }); + } + } + + let originated_mcast = ctx + .ctx + .db + .originated_mcast() + .map_err(|e| HttpError::for_internal_error(e.to_string()))?; + for origin in &originated_mcast { + let hop = MulticastPathHop::new( + ctx.ctx.hostname.clone(), + ctx.ctx.config.addr, + ); + multicast.insert(MulticastPathVector { + origin: origin.clone(), + path: vec![hop], + }); + } + + Ok(multicast) +} + +fn opt(s: HashSet) -> Option> { + if s.is_empty() { None } else { Some(s) } +} + +#[endpoint { method = GET, path = "/v3/pull" }] +async fn pull_handler_v3( + ctx: RequestContext>>, +) -> Result, HttpError> { + let ctx = ctx.context().lock().await.clone(); + let (underlay, tunnel) = collect_underlay_tunnel(&ctx)?; + Ok(HttpResponseOk(PullResponseV3 { + underlay: opt(underlay), + tunnel: opt(tunnel), + })) +} + +#[endpoint { method = GET, path = "/v4/pull" }] +async fn pull_handler_v4( + ctx: RequestContext>>, +) -> Result, HttpError> { + let ctx = ctx.context().lock().await.clone(); + let (underlay, tunnel) = collect_underlay_tunnel(&ctx)?; + let multicast = collect_multicast(&ctx)?; Ok(HttpResponseOk(PullResponse { - underlay: if underlay.is_empty() { - None - } else { - Some(underlay) - }, - tunnel: if tunnel.is_empty() { - None - } else { - Some(tunnel) - }, + underlay: opt(underlay), + tunnel: opt(tunnel), + multicast: opt(multicast), })) } @@ -831,6 +1018,10 @@ fn handle_update(update: &Update, ctx: &HandlerContext) { handle_tunnel_update(tunnel_update, ctx); } + if let Some(multicast_update) = &update.multicast { + handle_multicast_update(multicast_update, ctx); + } + // distribute updates if ctx.ctx.config.kind == RouterKind::Transit { @@ -846,13 +1037,24 @@ fn handle_update(update: &Update, ctx: &HandlerContext) { .as_ref() .map(|update| update.with_path_element(ctx.ctx.hostname.clone())); - let push = Update { + // Add our hop info to multicast path vectors before redistribution + let multicast = update.multicast.as_ref().map(|update| { + let hop = MulticastPathHop::new( + ctx.ctx.hostname.clone(), + ctx.ctx.config.addr, + ); + update.with_hop(hop) + }); + + let push = Arc::new(Update { underlay, tunnel: update.tunnel.clone(), - }; + multicast, + }); for ec in &ctx.ctx.event_channels { - ec.send(Event::Peer(PeerEvent::Push(push.clone()))).unwrap(); + ec.send(Event::Peer(PeerEvent::Push(Arc::clone(&push)))) + .unwrap(); } } } @@ -999,3 +1201,226 @@ fn handle_underlay_update(update: &UnderlayUpdate, ctx: &HandlerContext) { .imported_underlay_prefixes .store(ctx.ctx.db.imported_count() as u64, Ordering::Relaxed); } + +/// Handle multicast group subscription updates from a peer. +/// +/// Validation uses path-vector-based RPF rather than unicast-RIB RPF. +/// DDM operates on the underlay while multicast sources are overlay +/// addresses, so traditional (S,G) RPF against the unicast RIB does not +/// apply at this layer. The MRIB RPF module in rdb handles that check +/// before routes are originated into DDM. At the DDM exchange level, +/// the path vector provides loop detection and carries topology +/// information for replication optimization per [RFD 488]. +/// +/// [RFD 488]: https://rfd.shared.oxide.computer/rfd/0488 +fn handle_multicast_update(update: &MulticastUpdate, ctx: &HandlerContext) { + let db = &ctx.ctx.db; + let hostname = &ctx.ctx.hostname; + + let mut import = HashSet::new(); + for pv in &update.announce { + // Path-vector RPF: drop if our router_id appears in the path, + // indicating the announcement has already traversed us. + if pv.path.iter().any(|hop| &hop.router_id == hostname) { + dbg!( + ctx.log, + ctx.ctx.config.if_name, + "dropping multicast announce for {:?} - loop detected \ + (path length {})", + pv.origin.overlay_group, + pv.path.len(), + ); + continue; + } + + import.insert(MulticastRoute { + origin: pv.origin.clone(), + nexthop: ctx.peer, + path: pv.path.clone(), + }); + } + + let mut remove = HashSet::new(); + for pv in &update.withdraw { + // Empty path is safe: MulticastRoute's PartialEq/Hash exclude + // the path field, so this matches by (origin, nexthop) only. + remove.insert(MulticastRoute { + origin: pv.origin.clone(), + nexthop: ctx.peer, + path: Vec::new(), + }); + } + + // Atomic import + delete + diff under a single lock. + let (to_add, to_del) = db.update_imported_mcast(&import, &remove); + + if let Err(e) = crate::sys::add_multicast_routes( + &ctx.log, + &ctx.ctx.config.if_name, + &to_add, + ) { + err!( + ctx.log, + ctx.ctx.config.if_name, + "add multicast routes: {e}: {to_add:#?}", + ) + } + + if let Err(e) = crate::sys::remove_multicast_routes( + &ctx.log, + &ctx.ctx.config.if_name, + &to_del, + ) { + err!( + ctx.log, + ctx.ctx.config.if_name, + "remove multicast routes: {e}: {to_del:#?}", + ) + } +} + +#[cfg(test)] +mod test { + use super::*; + use ddm_types::exchange::MulticastPathHop; + use mg_common::net::{MulticastOrigin, UnderlayMulticastIpv6}; + use std::net::Ipv6Addr; + + fn sample_multicast_update() -> MulticastUpdate { + let origin = MulticastOrigin { + overlay_group: "233.252.0.1".parse().unwrap(), + underlay_group: UnderlayMulticastIpv6::new(Ipv6Addr::new( + 0xff04, 0, 0, 0, 0, 0, 0, 1, + )) + .unwrap(), + vni: 77, + metric: 0, + source: None, + }; + let pv = MulticastPathVector { + origin, + path: vec![MulticastPathHop::new( + "router-1".into(), + Ipv6Addr::LOCALHOST, + )], + }; + MulticastUpdate::announce([pv].into_iter().collect()) + } + + #[test] + fn v4_update_round_trips() { + let update = Update { + underlay: None, + tunnel: None, + multicast: Some(sample_multicast_update()), + }; + let json = serde_json::to_string(&update).unwrap(); + let back: Update = serde_json::from_str(&json).unwrap(); + assert!(back.multicast.is_some()); + assert_eq!(back.multicast.unwrap().announce.len(), 1,); + } + + #[test] + fn v4_update_deserializes_as_v3_drops_multicast() { + let update = Update { + underlay: None, + tunnel: None, + multicast: Some(sample_multicast_update()), + }; + let json = serde_json::to_string(&update).unwrap(); + // A V3 peer would deserialize this as UpdateV3, silently + // dropping the unknown multicast field. + let v3: UpdateV3 = serde_json::from_str(&json).unwrap(); + assert!(v3.underlay.is_none()); + assert!(v3.tunnel.is_none()); + } + + #[test] + fn v3_update_deserializes_as_v4_multicast_none() { + let v3 = UpdateV3 { + underlay: None, + tunnel: None, + }; + let json = serde_json::to_string(&v3).unwrap(); + // A V4 peer receiving a V3 update gets multicast: None. + let update: Update = serde_json::from_str(&json).unwrap(); + assert!(update.multicast.is_none()); + } + + #[test] + fn v4_pull_response_round_trips() { + let origin = MulticastOrigin { + overlay_group: "ff0e::1".parse().unwrap(), + underlay_group: UnderlayMulticastIpv6::new(Ipv6Addr::new( + 0xff04, 0, 0, 0, 0, 0, 0, 2, + )) + .unwrap(), + vni: 77, + metric: 0, + source: None, + }; + let pv = MulticastPathVector { + origin, + path: vec![], + }; + let resp = PullResponse { + underlay: None, + tunnel: None, + multicast: Some([pv].into_iter().collect()), + }; + let json = serde_json::to_string(&resp).unwrap(); + let back: PullResponse = serde_json::from_str(&json).unwrap(); + assert!(back.multicast.is_some()); + } + + #[test] + fn v4_pull_response_deserializes_as_v3() { + let origin = MulticastOrigin { + overlay_group: "233.252.0.1".parse().unwrap(), + underlay_group: UnderlayMulticastIpv6::new(Ipv6Addr::new( + 0xff04, 0, 0, 0, 0, 0, 0, 1, + )) + .unwrap(), + vni: 77, + metric: 0, + source: None, + }; + let pv = MulticastPathVector { + origin, + path: vec![], + }; + let resp = PullResponse { + underlay: None, + tunnel: None, + multicast: Some([pv].into_iter().collect()), + }; + let json = serde_json::to_string(&resp).unwrap(); + // V3 peer drops the multicast field. + let v3: PullResponseV3 = serde_json::from_str(&json).unwrap(); + assert!(v3.underlay.is_none()); + assert!(v3.tunnel.is_none()); + } + + #[test] + fn v3_pull_response_deserializes_as_v4() { + let v3 = PullResponseV3 { + underlay: None, + tunnel: None, + }; + let json = serde_json::to_string(&v3).unwrap(); + let resp: PullResponse = serde_json::from_str(&json).unwrap(); + assert!(resp.multicast.is_none()); + } + + #[test] + fn from_conversions_strip_multicast() { + let update = Update { + underlay: None, + tunnel: None, + multicast: Some(sample_multicast_update()), + }; + let v3 = UpdateV3::from(update); + let back = Update::from(v3); + assert!(back.multicast.is_none()); + } +} diff --git a/ddm/src/sm.rs b/ddm/src/sm.rs index 242157959..44e6c9165 100644 --- a/ddm/src/sm.rs +++ b/ddm/src/sm.rs @@ -4,12 +4,13 @@ use crate::db::Db; use crate::discovery::Version; -use crate::exchange::{TunnelUpdate, UnderlayUpdate, Update}; +use crate::exchange::{MulticastUpdate, TunnelUpdate, UnderlayUpdate, Update}; use crate::{dbg, discovery, err, exchange, inf, wrn}; use ddm_types::db::RouterKind; +use ddm_types::exchange::MulticastPathHop; use ddm_types::exchange::PathVector; use libnet::get_ipaddr_info; -use mg_common::net::TunnelOrigin; +use mg_common::net::{MulticastOrigin, TunnelOrigin}; use oxnet::Ipv6Net; use slog::Logger; use std::collections::HashSet; @@ -41,11 +42,12 @@ pub enum AdminEvent { pub enum PrefixSet { Underlay(HashSet), Tunnel(HashSet), + Multicast(HashSet), } #[derive(Debug)] pub enum PeerEvent { - Push(Update), + Push(Arc), } #[derive(Debug)] @@ -425,7 +427,7 @@ impl Exchange { ); let interval = 250; // TODO as parameter loop { - match exchange::do_pull( + match exchange::do_pull_v4( &self.ctx, &self.ctx.config.addr, &self.ctx.rt, @@ -455,7 +457,7 @@ impl Exchange { ) { exchange_thread.abort(); self.ctx.db.remove_peer(self.ctx.config.if_index); - let (to_remove, to_remove_tnl) = + let (to_remove, to_remove_tnl, to_remove_mcast) = self.ctx.db.remove_nexthop_routes(self.peer); let mut routes: Vec = Vec::new(); for x in &to_remove { @@ -518,9 +520,33 @@ impl Exchange { )) }; - let push = Update { underlay, tunnel }; + // Build multicast withdrawal with our hop info + let multicast = if to_remove_mcast.is_empty() { + None + } else { + let hop = MulticastPathHop::new( + self.ctx.hostname.clone(), + self.ctx.config.addr, + ); + Some(MulticastUpdate::withdraw( + to_remove_mcast + .iter() + .map(|route| ddm_types::exchange::MulticastPathVector { + origin: route.origin.clone(), + path: vec![hop.clone()], + }) + .collect(), + )) + }; + + let push = Arc::new(Update { + underlay, + tunnel, + multicast, + }); for ec in &self.ctx.event_channels { - ec.send(Event::Peer(PeerEvent::Push(push.clone()))).unwrap(); + ec.send(Event::Peer(PeerEvent::Push(Arc::clone(&push)))) + .unwrap(); } } pull_stop.store(true, Ordering::Relaxed); @@ -601,6 +627,7 @@ impl State for Exchange { "announce: {}", e, ); + wrn!( self.log, self.ctx.config.if_name, @@ -728,6 +755,104 @@ impl State for Exchange { ); } } + Event::Admin(AdminEvent::Announce(PrefixSet::Multicast( + groups, + ))) => { + // Convert `MulticastOrigin` to `MulticastPathVector` with + // our hop info + let hop = MulticastPathHop::new( + self.ctx.hostname.clone(), + self.ctx.config.addr, + ); + let pvs: HashSet<_> = groups + .iter() + .map(|origin| { + ddm_types::exchange::MulticastPathVector { + origin: origin.clone(), + path: vec![hop.clone()], + } + }) + .collect(); + + if let Err(e) = crate::exchange::announce_multicast( + &self.ctx, + self.ctx.config.clone(), + pvs, + self.peer, + self.version, + self.ctx.rt.clone(), + self.log.clone(), + ) { + err!( + self.log, + self.ctx.config.if_name, + "announce multicast: {}", + e, + ); + wrn!( + self.log, + self.ctx.config.if_name, + "expiring peer {} due to failed multicast announce", + self.peer, + ); + self.expire_peer(&exchange_thread, &pull_stop); + return ( + Box::new(Solicit::new( + self.ctx.clone(), + self.log.clone(), + )), + event, + ); + } + } + Event::Admin(AdminEvent::Withdraw(PrefixSet::Multicast( + groups, + ))) => { + // Convert MulticastOrigin to MulticastPathVector for withdrawal + let hop = MulticastPathHop::new( + self.ctx.hostname.clone(), + self.ctx.config.addr, + ); + let pvs: HashSet<_> = groups + .iter() + .map(|origin| { + ddm_types::exchange::MulticastPathVector { + origin: origin.clone(), + path: vec![hop.clone()], + } + }) + .collect(); + + if let Err(e) = crate::exchange::withdraw_multicast( + &self.ctx, + self.ctx.config.clone(), + pvs, + self.peer, + self.version, + self.ctx.rt.clone(), + self.log.clone(), + ) { + err!( + self.log, + self.ctx.config.if_name, + "withdraw multicast: {e}", + ); + wrn!( + self.log, + self.ctx.config.if_name, + "expiring peer {} due to failed multicast withdraw", + self.peer, + ); + self.expire_peer(&exchange_thread, &pull_stop); + return ( + Box::new(Solicit::new( + self.ctx.clone(), + self.log.clone(), + )), + event, + ); + } + } Event::Admin(AdminEvent::Expire(peer)) => { if self.peer == peer { inf!( @@ -770,6 +895,8 @@ impl State for Exchange { self.peer, update, ); + let update = Arc::try_unwrap(update) + .unwrap_or_else(|arc| (*arc).clone()); if let Some(push) = update.underlay { if !push.announce.is_empty() && let Err(e) = crate::exchange::announce_underlay( @@ -817,8 +944,7 @@ impl State for Exchange { err!( self.log, self.ctx.config.if_name, - "withdraw: {}", - e, + "withdraw: {e}", ); wrn!( self.log, @@ -836,6 +962,71 @@ impl State for Exchange { ); } } + // Handle multicast redistribution + if let Some(push) = update.multicast { + if !push.announce.is_empty() + && let Err(e) = crate::exchange::announce_multicast( + &self.ctx, + self.ctx.config.clone(), + push.announce, + self.peer, + self.version, + self.ctx.rt.clone(), + self.log.clone(), + ) + { + err!( + self.log, + self.ctx.config.if_name, + "announce multicast: {e}", + ); + wrn!( + self.log, + self.ctx.config.if_name, + "expiring peer {} due to failed multicast announce", + self.peer, + ); + self.expire_peer(&exchange_thread, &pull_stop); + return ( + Box::new(Solicit::new( + self.ctx.clone(), + self.log.clone(), + )), + event, + ); + } + if !push.withdraw.is_empty() + && let Err(e) = crate::exchange::withdraw_multicast( + &self.ctx, + self.ctx.config.clone(), + push.withdraw, + self.peer, + self.version, + self.ctx.rt.clone(), + self.log.clone(), + ) + { + err!( + self.log, + self.ctx.config.if_name, + "withdraw multicast: {e}", + ); + wrn!( + self.log, + self.ctx.config.if_name, + "expiring peer {} due to failed multicast withdraw", + self.peer, + ); + self.expire_peer(&exchange_thread, &pull_stop); + return ( + Box::new(Solicit::new( + self.ctx.clone(), + self.log.clone(), + )), + event, + ); + } + } } Event::Neighbor(NeighborEvent::Expire) => { wrn!( diff --git a/ddm/src/sys.rs b/ddm/src/sys.rs index 915fc0b54..22b782f0a 100644 --- a/ddm/src/sys.rs +++ b/ddm/src/sys.rs @@ -4,7 +4,7 @@ use crate::sm::{Config, DpdConfig}; use crate::{dbg, err, inf, wrn}; -use ddm_types::db::TunnelRoute; +use ddm_types::db::{MulticastRoute, TunnelRoute}; use dpd_client::Client; use dpd_client::ClientState; use dpd_client::types; @@ -359,6 +359,99 @@ pub fn remove_tunnel_routes( Ok(()) } +#[cfg(not(target_os = "illumos"))] +pub fn add_multicast_routes( + _log: &Logger, + _ifname: &str, + _routes: &HashSet, +) -> Result<(), String> { + todo!(); +} + +/// Update OPTE multicast-to-physical (M2P) table entries for learned +/// multicast routes. Each route's overlay group is mapped to the +/// corresponding underlay multicast address so that OPTE can direct +/// multicast traffic to the correct underlay destinations. +#[cfg(target_os = "illumos")] +pub fn add_multicast_routes( + log: &Logger, + ifname: &str, + routes: &HashSet, +) -> Result<(), String> { + use oxide_vpc::api::MulticastUnderlay; + use oxide_vpc::api::SetMcast2PhysReq; + + let hdl = OpteHdl::open().map_err(|e| e.to_string())?; + + for route in routes { + let underlay = + MulticastUnderlay::new(route.origin.underlay_group.ip().into()) + .map_err(|e| { + format!( + "invalid underlay multicast address {}: {e}", + route.origin.underlay_group, + ) + })?; + let req = SetMcast2PhysReq { + group: route.origin.overlay_group.into(), + underlay, + }; + let overlay = route.origin.overlay_group; + let underlay_addr = route.origin.underlay_group; + inf!(log, ifname, "adding M2P: {overlay:?} -> {underlay_addr}"); + if let Err(e) = hdl.set_m2p(&req) { + err!(log, ifname, "failed to set M2P route: {req:?}: {e}"); + } + } + + Ok(()) +} + +#[cfg(not(target_os = "illumos"))] +pub fn remove_multicast_routes( + _log: &Logger, + _ifname: &str, + _routes: &HashSet, +) -> Result<(), String> { + todo!() +} + +/// Remove OPTE M2P table entries for withdrawn multicast routes. +#[cfg(target_os = "illumos")] +pub fn remove_multicast_routes( + log: &Logger, + ifname: &str, + routes: &HashSet, +) -> Result<(), String> { + use oxide_vpc::api::ClearMcast2PhysReq; + use oxide_vpc::api::MulticastUnderlay; + + let hdl = OpteHdl::open().map_err(|e| e.to_string())?; + + for route in routes { + let underlay = + MulticastUnderlay::new(route.origin.underlay_group.ip().into()) + .map_err(|e| { + format!( + "invalid underlay multicast address {}: {e}", + route.origin.underlay_group, + ) + })?; + let req = ClearMcast2PhysReq { + group: route.origin.overlay_group.into(), + underlay, + }; + let overlay = route.origin.overlay_group; + let underlay_addr = route.origin.underlay_group; + inf!(log, ifname, "removing M2P: {overlay:?} -> {underlay_addr}"); + if let Err(e) = hdl.clear_m2p(&req) { + err!(log, ifname, "failed to clear M2P route: {req:?}: {e}"); + } + } + + Ok(()) +} + pub fn remove_underlay_routes( log: &Logger, ifname: &str, diff --git a/ddmadm/src/main.rs b/ddmadm/src/main.rs index 800315d86..309b8a6f3 100644 --- a/ddmadm/src/main.rs +++ b/ddmadm/src/main.rs @@ -60,6 +60,18 @@ enum SubCommand { /// Withdraw prefixes from a DDM router. TunnelWithdraw(TunnelEndpoint), + /// Get multicast groups imported from DDM peers. + MulticastImported, + + /// Get locally originated multicast groups. + MulticastOriginated, + + /// Advertise multicast groups from this router. + MulticastAdvertise(MulticastGroup), + + /// Withdraw multicast groups from this router. + MulticastWithdraw(MulticastGroup), + /// Sync prefix information from peers. Sync, } @@ -84,6 +96,29 @@ struct TunnelEndpoint { pub metric: u64, } +#[derive(Debug, Parser)] +struct MulticastGroup { + /// Overlay multicast group address (e.g. 233.252.0.1 or ff0e::1). + #[arg(short = 'g', long)] + pub overlay_group: IpAddr, + + /// Underlay multicast address (ff04::/64 admin-local scope). + #[arg(short = 'u', long)] + pub underlay_group: Ipv6Addr, + + /// Virtual Network Identifier. + #[arg(short, long)] + pub vni: u32, + + /// Path metric. + #[arg(short, long, default_value_t = 0)] + pub metric: u64, + + /// Source address for (S,G) routes (omit for (*,G)). + #[arg(short, long)] + pub source: Option, +} + #[derive(Debug, Parser)] struct Peer { addr: Ipv6Addr, @@ -242,6 +277,94 @@ async fn run() -> Result<()> { }]) .await?; } + SubCommand::MulticastImported => { + let msg = client.get_multicast_groups().await?; + let mut tw = TabWriter::new(stdout()); + writeln!( + &mut tw, + "{}\t{}\t{}\t{}\t{}\t{}", + "Overlay Group".dimmed(), + "Underlay Group".dimmed(), + "VNI".dimmed(), + "Metric".dimmed(), + "Source".dimmed(), + "Path".dimmed(), + )?; + for route in msg.into_inner() { + let source = match &route.origin.source { + Some(s) => s.to_string(), + None => "(*,G)".to_string(), + }; + let path: Vec<_> = route + .path + .iter() + .rev() + .map(|h| h.router_id.clone()) + .collect(); + writeln!( + &mut tw, + "{}\t{}\t{}\t{}\t{}\t{}", + route.origin.overlay_group, + route.origin.underlay_group, + route.origin.vni, + route.origin.metric, + source, + path.join(" "), + )?; + } + tw.flush()?; + } + SubCommand::MulticastOriginated => { + let msg = client.get_originated_multicast_groups().await?; + let mut tw = TabWriter::new(stdout()); + writeln!( + &mut tw, + "{}\t{}\t{}\t{}\t{}", + "Overlay Group".dimmed(), + "Underlay Group".dimmed(), + "VNI".dimmed(), + "Metric".dimmed(), + "Source".dimmed(), + )?; + for origin in msg.into_inner() { + let source = match &origin.source { + Some(s) => s.to_string(), + None => "(*,G)".to_string(), + }; + writeln!( + &mut tw, + "{}\t{}\t{}\t{}\t{}", + origin.overlay_group, + origin.underlay_group, + origin.vni, + origin.metric, + source, + )?; + } + tw.flush()?; + } + SubCommand::MulticastAdvertise(mg) => { + client + .advertise_multicast_groups(&vec![types::MulticastOrigin { + overlay_group: mg.overlay_group, + underlay_group: mg.underlay_group, + vni: mg.vni, + metric: mg.metric, + source: mg.source, + }]) + .await?; + } + SubCommand::MulticastWithdraw(mg) => { + client + .withdraw_multicast_groups(&vec![types::MulticastOrigin { + overlay_group: mg.overlay_group, + underlay_group: mg.underlay_group, + vni: mg.vni, + metric: mg.metric, + source: mg.source, + }]) + .await?; + } SubCommand::Sync => { client.sync().await?; } diff --git a/mg-common/Cargo.toml b/mg-common/Cargo.toml index 87a303084..0c40e3174 100644 --- a/mg-common/Cargo.toml +++ b/mg-common/Cargo.toml @@ -19,12 +19,16 @@ backoff.workspace = true smf.workspace = true uuid.workspace = true libc.workspace = true +omicron-common.workspace = true # We need this on illumos, but must omit it on other platforms [target.'cfg(target_os = "illumos")'.dependencies.libnet] workspace = true optional = true +[dev-dependencies] +serde_json.workspace = true + [features] default = ["libnet"] libnet = ["dep:libnet"] diff --git a/mg-common/src/net.rs b/mg-common/src/net.rs index f1784afee..bce080b29 100644 --- a/mg-common/src/net.rs +++ b/mg-common/src/net.rs @@ -2,10 +2,102 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +use omicron_common::address::UNDERLAY_MULTICAST_SUBNET; +use omicron_common::api::external::Vni; use oxnet::{IpNet, Ipv4Net, Ipv6Net}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use std::net::{Ipv4Addr, Ipv6Addr}; +use std::fmt; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; +use std::str::FromStr; + +/// Default VNI for multicast routing. +pub const DEFAULT_MULTICAST_VNI: u32 = Vni::DEFAULT_MULTICAST_VNI.as_u32(); + +fn default_multicast_vni() -> u32 { + DEFAULT_MULTICAST_VNI +} + +/// A validated underlay multicast IPv6 address within ff04::/64. +/// +/// The Oxide rack maps overlay multicast groups 1:1 to admin-local scoped +/// IPv6 multicast addresses in `UNDERLAY_MULTICAST_SUBNET` (ff04::/64). +/// This type enforces that invariant at construction time. +#[derive( + Debug, + Copy, + Clone, + Eq, + PartialEq, + PartialOrd, + Ord, + Hash, + Serialize, + Deserialize, + JsonSchema, +)] +#[serde(try_from = "Ipv6Addr", into = "Ipv6Addr")] +#[schemars(transparent)] +pub struct UnderlayMulticastIpv6(Ipv6Addr); + +impl UnderlayMulticastIpv6 { + /// Create a new validated underlay multicast address. + /// + /// # Errors + /// + /// Returns an error if the address is not within ff04::/64. + pub fn new(value: Ipv6Addr) -> Result { + if !UNDERLAY_MULTICAST_SUBNET.contains(value) { + return Err(format!( + "underlay address {value} is not within \ + {UNDERLAY_MULTICAST_SUBNET}" + )); + } + Ok(Self(value)) + } + + /// Returns the underlying IPv6 address. + #[inline] + pub const fn ip(&self) -> Ipv6Addr { + self.0 + } +} + +impl fmt::Display for UnderlayMulticastIpv6 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +impl TryFrom for UnderlayMulticastIpv6 { + type Error = String; + + fn try_from(value: Ipv6Addr) -> Result { + Self::new(value) + } +} + +impl From for Ipv6Addr { + fn from(addr: UnderlayMulticastIpv6) -> Self { + addr.0 + } +} + +impl From for IpAddr { + fn from(addr: UnderlayMulticastIpv6) -> Self { + IpAddr::V6(addr.0) + } +} + +impl FromStr for UnderlayMulticastIpv6 { + type Err = String; + + fn from_str(s: &str) -> Result { + let addr: Ipv6Addr = + s.parse().map_err(|e| format!("invalid IPv6: {e}"))?; + Self::new(addr) + } +} #[derive( Debug, Copy, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema, @@ -93,3 +185,131 @@ pub enum IpPrefix { V4(Ipv4Prefix), V6(Ipv6Prefix), } + +/// Origin information for a multicast group announcement. +/// +/// This is analogous to TunnelOrigin but for multicast groups. +/// +/// This represents a subscription to a multicast group that should be +/// advertised via DDM. The overlay_group is the application-visible multicast +/// address (e.g., 233.252.0.1 or ff0e::1), while underlay_group is the mapped +/// admin-local scoped IPv6 address (ff04::X) used in the underlay network. +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct MulticastOrigin { + /// The overlay multicast group address (IPv4 or IPv6). + /// This is the group address visible to applications. + pub overlay_group: IpAddr, + + /// The underlay multicast group address (ff04::X). + /// Validated at construction to be within ff04::/64. + pub underlay_group: UnderlayMulticastIpv6, + + /// VNI for this multicast group (identifies the VPC/network context). + #[serde(default = "default_multicast_vni")] + pub vni: u32, + + /// Metric for path selection (lower is better). + /// + /// Used for multi-rack replication optimization. + /// Excluded from identity (Hash/Eq) so that metric changes update + /// an existing entry rather than creating a duplicate. + #[serde(default)] + pub metric: u64, + + /// Optional source address for Source-Specific Multicast (S,G) routes. + /// None for Any-Source Multicast (*,G) routes. + #[serde(default)] + pub source: Option, +} + +impl PartialEq for MulticastOrigin { + fn eq(&self, other: &Self) -> bool { + self.overlay_group == other.overlay_group + && self.underlay_group == other.underlay_group + && self.vni == other.vni + && self.source == other.source + } +} + +impl Eq for MulticastOrigin {} + +impl std::hash::Hash for MulticastOrigin { + fn hash(&self, state: &mut H) { + self.overlay_group.hash(state); + self.underlay_group.hash(state); + self.vni.hash(state); + self.source.hash(state); + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn underlay_valid_ff04() { + let addr = Ipv6Addr::new(0xff04, 0, 0, 0, 0, 0, 0, 1); + assert!(UnderlayMulticastIpv6::new(addr).is_ok()); + } + + #[test] + fn underlay_rejects_non_admin_local() { + // ff0e:: is global scope, not admin-local + let addr = Ipv6Addr::new(0xff0e, 0, 0, 0, 0, 0, 0, 1); + assert!(UnderlayMulticastIpv6::new(addr).is_err()); + } + + #[test] + fn underlay_rejects_unicast() { + let addr = Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 1); + assert!(UnderlayMulticastIpv6::new(addr).is_err()); + } + + #[test] + fn underlay_serde_round_trip() { + let addr = UnderlayMulticastIpv6::new(Ipv6Addr::new( + 0xff04, 0, 0, 0, 0, 0, 0, 42, + )) + .unwrap(); + let json = serde_json::to_string(&addr).unwrap(); + let back: UnderlayMulticastIpv6 = serde_json::from_str(&json).unwrap(); + assert_eq!(addr, back); + } + + #[test] + fn underlay_serde_rejects_invalid() { + // ff0e::1 serialized as an Ipv6Addr, then deserialized as + // UnderlayMulticastIpv6 should fail via try_from. + let json = + serde_json::to_string(&Ipv6Addr::new(0xff0e, 0, 0, 0, 0, 0, 0, 1)) + .unwrap(); + let result: Result = + serde_json::from_str(&json); + assert!(result.is_err()); + } + + #[test] + fn multicast_origin_rejects_bad_underlay() { + let json = serde_json::json!({ + "overlay_group": "233.252.0.1", + "underlay_group": "ff0e::1", + "vni": 77 + }); + let result: Result = serde_json::from_value(json); + assert!(result.is_err()); + } + + #[test] + fn multicast_origin_accepts_valid() { + let json = serde_json::json!({ + "overlay_group": "233.252.0.1", + "underlay_group": "ff04::1", + "vni": 77 + }); + let origin: MulticastOrigin = serde_json::from_value(json).unwrap(); + assert_eq!( + origin.underlay_group.ip(), + Ipv6Addr::new(0xff04, 0, 0, 0, 0, 0, 0, 1), + ); + } +} diff --git a/mg-lower/src/ddm.rs b/mg-lower/src/ddm.rs index ac7d97086..45a530233 100644 --- a/mg-lower/src/ddm.rs +++ b/mg-lower/src/ddm.rs @@ -5,7 +5,7 @@ use crate::log::ddm_log; #[cfg(target_os = "illumos")] use ddm_admin_client::Client; -use ddm_admin_client::types::TunnelOrigin; +use ddm_admin_client::types::{MulticastOrigin, TunnelOrigin}; use oxnet::Ipv6Net; use slog::Logger; use std::{net::Ipv6Addr, sync::Arc}; @@ -111,3 +111,57 @@ pub(crate) fn remove_tunnel_routes<'a, I: Iterator>( pub fn new_ddm_client(log: &Logger) -> Client { Client::new("http://localhost:8000", log.clone()) } + +pub(crate) fn add_multicast_routes< + 'a, + I: Iterator, +>( + client: &impl Ddm, + routes: I, + rt: &Arc, + log: &Logger, +) { + let routes: Vec = routes.cloned().collect(); + if routes.is_empty() { + return; + } + let resp = + rt.block_on(async { client.advertise_multicast_groups(&routes).await }); + if let Err(e) = resp { + ddm_log!(log, + error, + "advertise multicast groups error: {e}"; + "error" => format!("{e}"), + "groups" => format!("{routes:#?}") + ); + } +} + +pub(crate) fn remove_multicast_routes< + 'a, + I: Iterator, +>( + client: &impl Ddm, + routes: I, + rt: &Arc, + log: &Logger, +) { + let routes: Vec = routes.cloned().collect(); + if routes.is_empty() { + return; + } + let resp = + rt.block_on(async { client.withdraw_multicast_groups(&routes).await }); + match resp { + Err(e) => ddm_log!(log, + error, + "withdraw multicast groups error: {e}"; + "groups" => format!("{routes:#?}") + ), + Ok(_) => ddm_log!(log, + info, + "withdrew multicast groups"; + "groups" => format!("{routes:#?}") + ), + } +} diff --git a/mg-lower/src/lib.rs b/mg-lower/src/lib.rs index bcbf5b57e..c1b84595f 100644 --- a/mg-lower/src/lib.rs +++ b/mg-lower/src/lib.rs @@ -39,6 +39,7 @@ mod ddm; mod dendrite; mod error; mod log; +pub mod mrib; mod platform; #[cfg(test)] diff --git a/mg-lower/src/mrib.rs b/mg-lower/src/mrib.rs new file mode 100644 index 000000000..ea8069e92 --- /dev/null +++ b/mg-lower/src/mrib.rs @@ -0,0 +1,234 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! MRIB (Multicast Routing Information Base) synchronization to DDM. +//! +//! This module watches for MRIB changes and propagates multicast group +//! subscriptions to DDM for distribution across the underlay network. +//! +//! ## Data Flow +//! +//! ```text +//! MRIB (loc_mrib changes) +//! | +//! v [MribChangeNotification] +//! mg-lower/mrib.rs +//! | +//! v [MulticastOrigin] +//! DDM admin API +//! | +//! v [DDM exchange protocol] +//! Other sleds/racks +//! ``` + +use crate::ddm::{ + add_multicast_routes, new_ddm_client, remove_multicast_routes, +}; +use crate::platform::{Ddm, ProductionDdm}; +use ddm_admin_client::types::MulticastOrigin; +use mg_common::net::DEFAULT_MULTICAST_VNI; +use rdb::Mrib; +use rdb::types::{ + DEFAULT_MULTICAST_VNI as DEFAULT_MCAST_VNI, MribChangeNotification, + MulticastAddr, MulticastRoute, +}; +use slog::{Logger, debug, error, info}; +use std::collections::HashSet; +use std::net::IpAddr; +use std::sync::Arc; +use std::sync::mpsc::{RecvTimeoutError, channel}; +use std::thread::sleep; +use std::time::Duration; + +const MG_LOWER_MRIB_TAG: &str = "mg-lower-mrib"; + +/// Convert an MRIB MulticastRoute to a DDM MulticastOrigin. +/// +/// The MulticastOrigin captures the essential information needed for DDM +/// to advertise the multicast group subscription to other routers: +/// - overlay_group: The multicast group address (e.g., 233.252.0.1 or ff0e::1) +/// - underlay_group: The mapped underlay address within the ff04::/64 subnet +/// (a /64 within admin-local scope per RFC 7346, reserved for the rack's +/// underlay multicast traffic) +/// - source: Optional source for (S,G) routes, None for (*,G) +/// - vni: Virtual Network Identifier (default multicast VNI) +fn mrib_route_to_ddm_origin(route: &MulticastRoute) -> MulticastOrigin { + // Extract overlay group address from the route key + let overlay_group: IpAddr = match route.key.group() { + MulticastAddr::V4(v4) => IpAddr::V4(v4.ip()), + MulticastAddr::V6(v6) => IpAddr::V6(v6.ip()), + }; + + // Extract source address for (S,G) routes + let source = route.key.source(); + + MulticastOrigin { + overlay_group, + underlay_group: route.underlay_group.into(), + vni: DEFAULT_MULTICAST_VNI, + metric: 0, // Default metric + source, + } +} + +/// Run the MRIB synchronization loop. +/// +/// This function loops forever, watching for MRIB changes and synchronizing +/// them to DDM. It runs on the calling thread. +pub fn run(mrib: Mrib, log: Logger, rt: Arc) { + loop { + let (tx, rx) = channel(); + + // Register as MRIB watcher + mrib.watch(MG_LOWER_MRIB_TAG.into(), tx); + + let ddm = ProductionDdm { + client: new_ddm_client(&log), + }; + + // Initial full sync + if let Err(e) = full_sync(&mrib, &ddm, &log, &rt) { + error!(log, "MRIB full sync failed: {e}"); + info!(log, "restarting MRIB sync loop in one second"); + sleep(Duration::from_secs(1)); + continue; + } + + // Handle incremental changes + loop { + match rx.recv_timeout(Duration::from_secs(10)) { + Ok(notification) => { + if let Err(e) = + handle_change(&mrib, notification, &ddm, &log, &rt) + { + error!(log, "MRIB change handling failed: {e}"); + } + } + Err(RecvTimeoutError::Timeout) => { + // Periodic full sync to catch any missed changes + if let Err(e) = full_sync(&mrib, &ddm, &log, &rt) { + error!(log, "MRIB periodic sync failed: {e}"); + } + } + Err(RecvTimeoutError::Disconnected) => { + error!(log, "MRIB watcher disconnected"); + break; + } + } + } + } +} + +/// Perform a full synchronization of MRIB to DDM. +/// +/// This compares the current MRIB loc_mrib with what DDM has advertised +/// and reconciles any differences. +pub(crate) fn full_sync( + mrib: &Mrib, + ddm: &D, + log: &Logger, + rt: &Arc, +) -> Result<(), String> { + // Get current MRIB state (installed/selected routes) + let mrib_routes = mrib.loc_mrib(); + + // Convert to DDM MulticastOrigin set + let mrib_origins: HashSet = + mrib_routes.values().map(mrib_route_to_ddm_origin).collect(); + + // Get current DDM advertised state + let ddm_current: HashSet = rt + .block_on(async { ddm.get_originated_multicast_groups().await }) + .map_err(|e| format!("failed to get DDM multicast groups: {e}"))? + .into_inner() + .into_iter() + .collect(); + + // Compute diff + let to_add: Vec<_> = mrib_origins.difference(&ddm_current).collect(); + let to_remove: Vec<_> = ddm_current.difference(&mrib_origins).collect(); + + if !to_add.is_empty() { + info!( + log, + "MRIB sync: adding {} multicast groups to DDM", + to_add.len() + ); + add_multicast_routes(ddm, to_add.into_iter(), rt, log); + } + + if !to_remove.is_empty() { + info!( + log, + "MRIB sync: removing {} multicast groups from DDM", + to_remove.len() + ); + remove_multicast_routes(ddm, to_remove.into_iter(), rt, log); + } + + Ok(()) +} + +/// Handle an incremental MRIB change notification. +fn handle_change( + mrib: &Mrib, + notification: MribChangeNotification, + ddm: &D, + log: &Logger, + rt: &Arc, +) -> Result<(), String> { + // Get current DDM state for comparison + let ddm_current: HashSet = rt + .block_on(async { ddm.get_originated_multicast_groups().await }) + .map_err(|e| format!("failed to get DDM multicast groups: {e}"))? + .into_inner() + .into_iter() + .collect(); + + let mut to_add = Vec::new(); + let mut to_remove = Vec::new(); + + for key in notification.changed { + // Check if route exists in loc_mrib (installed) + if let Some(route) = mrib.get_selected_route(&key) { + let origin = mrib_route_to_ddm_origin(&route); + if !ddm_current.contains(&origin) { + to_add.push(origin); + } + } else { + // Route was removed from loc_mrib, so we need to find matching DDM + // origin. We check all DDM origins to find any that match this key + for ddm_origin in &ddm_current { + // Reconstruct the key from the DDM origin to compare + if let Ok(overlay_group) = + MulticastAddr::try_from(ddm_origin.overlay_group) + && let Ok(ddm_key) = rdb::types::MulticastRouteKey::new( + ddm_origin.source, + overlay_group, + DEFAULT_MCAST_VNI, + ) + && ddm_key == key + { + to_remove.push(ddm_origin.clone()); + } + } + } + } + + if !to_add.is_empty() { + debug!(log, "MRIB change: adding {} multicast groups", to_add.len()); + add_multicast_routes(ddm, to_add.iter(), rt, log); + } + + if !to_remove.is_empty() { + debug!( + log, + "MRIB change: removing {} multicast groups", + to_remove.len() + ); + remove_multicast_routes(ddm, to_remove.iter(), rt, log); + } + + Ok(()) +} diff --git a/mg-lower/src/platform.rs b/mg-lower/src/platform.rs index a05143b9b..363dceafe 100644 --- a/mg-lower/src/platform.rs +++ b/mg-lower/src/platform.rs @@ -216,6 +216,31 @@ pub trait Ddm { ddm_admin_client::ResponseValue<()>, ddm_admin_client::Error, >; + + async fn get_originated_multicast_groups( + &self, + ) -> Result< + ddm_admin_client::ResponseValue>, + ddm_admin_client::Error, + >; + + #[allow(clippy::ptr_arg)] + async fn advertise_multicast_groups<'a>( + &'a self, + body: &'a Vec, + ) -> Result< + ddm_admin_client::ResponseValue<()>, + ddm_admin_client::Error, + >; + + #[allow(clippy::ptr_arg)] + async fn withdraw_multicast_groups<'a>( + &'a self, + body: &'a Vec, + ) -> Result< + ddm_admin_client::ResponseValue<()>, + ddm_admin_client::Error, + >; } /// This trait wraps the methods that have expectations about switch zone @@ -405,6 +430,35 @@ impl Ddm for ProductionDdm { > { self.client.withdraw_tunnel_endpoints(body).await } + + async fn get_originated_multicast_groups( + &self, + ) -> Result< + ddm_admin_client::ResponseValue>, + ddm_admin_client::Error, + > { + self.client.get_originated_multicast_groups().await + } + + async fn advertise_multicast_groups<'a>( + &'a self, + body: &'a Vec, + ) -> Result< + ddm_admin_client::ResponseValue<()>, + ddm_admin_client::Error, + > { + self.client.advertise_multicast_groups(body).await + } + + async fn withdraw_multicast_groups<'a>( + &'a self, + body: &'a Vec, + ) -> Result< + ddm_admin_client::ResponseValue<()>, + ddm_admin_client::Error, + > { + self.client.withdraw_multicast_groups(body).await + } } /// Production switch zone that uses libnet for route lookups (illumos only). @@ -699,6 +753,7 @@ pub(crate) mod test { pub(crate) struct TestDdm { pub(crate) tunnel_originated: Mutex>, pub(crate) originated: Mutex>, + pub(crate) multicast_originated: Mutex>, } impl Default for TestDdm { @@ -706,6 +761,7 @@ pub(crate) mod test { Self { tunnel_originated: Mutex::new(Vec::default()), originated: Mutex::new(Vec::default()), + multicast_originated: Mutex::new(Vec::default()), } } } @@ -766,6 +822,45 @@ pub(crate) mod test { .retain(|x| !body.contains(x)); Ok(ddm_response_ok!(())) } + + async fn get_originated_multicast_groups( + &self, + ) -> Result< + ddm_admin_client::ResponseValue>, + ddm_admin_client::Error, + > { + Ok(ddm_response_ok!( + self.multicast_originated.lock().unwrap().clone() + )) + } + + async fn advertise_multicast_groups<'a>( + &'a self, + body: &'a Vec, + ) -> Result< + ddm_admin_client::ResponseValue<()>, + ddm_admin_client::Error, + > { + self.multicast_originated + .lock() + .unwrap() + .extend(body.clone()); + Ok(ddm_response_ok!(())) + } + + async fn withdraw_multicast_groups<'a>( + &'a self, + body: &'a Vec, + ) -> Result< + ddm_admin_client::ResponseValue<()>, + ddm_admin_client::Error, + > { + self.multicast_originated + .lock() + .unwrap() + .retain(|x| !body.contains(x)); + Ok(ddm_response_ok!(())) + } } /// A mock switch zone implementation. diff --git a/openapi/ddm-admin/ddm-admin-1.0.0-b6eac7.json.gitstub b/openapi/ddm-admin/ddm-admin-1.0.0-b6eac7.json.gitstub new file mode 100644 index 000000000..0d935c8bd --- /dev/null +++ b/openapi/ddm-admin/ddm-admin-1.0.0-b6eac7.json.gitstub @@ -0,0 +1 @@ +76204d2907209bd8b963fb2da976ea688282d990:openapi/ddm-admin/ddm-admin-1.0.0-b6eac7.json diff --git a/openapi/ddm-admin/ddm-admin-1.0.0-b6eac7.json b/openapi/ddm-admin/ddm-admin-2.0.0-3dc476.json similarity index 64% rename from openapi/ddm-admin/ddm-admin-1.0.0-b6eac7.json rename to openapi/ddm-admin/ddm-admin-2.0.0-3dc476.json index fe80efd35..954fd3ad4 100644 --- a/openapi/ddm-admin/ddm-admin-1.0.0-b6eac7.json +++ b/openapi/ddm-admin/ddm-admin-2.0.0-3dc476.json @@ -6,7 +6,7 @@ "url": "https://oxide.computer", "email": "api@oxide.computer" }, - "version": "1.0.0" + "version": "2.0.0" }, "paths": { "/disable-stats": { @@ -51,6 +51,94 @@ } } }, + "/multicast_group": { + "put": { + "operationId": "advertise_multicast_groups", + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Set_of_MulticastOrigin", + "type": "array", + "items": { + "$ref": "#/components/schemas/MulticastOrigin" + }, + "uniqueItems": true + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "operationId": "withdraw_multicast_groups", + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Set_of_MulticastOrigin", + "type": "array", + "items": { + "$ref": "#/components/schemas/MulticastOrigin" + }, + "uniqueItems": true + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/multicast_groups": { + "get": { + "operationId": "get_multicast_groups", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Set_of_MulticastRoute", + "type": "array", + "items": { + "$ref": "#/components/schemas/MulticastRoute" + }, + "uniqueItems": true + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/originated": { "get": { "operationId": "get_originated", @@ -79,6 +167,34 @@ } } }, + "/originated_multicast_groups": { + "get": { + "operationId": "get_originated_multicast_groups", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Set_of_MulticastOrigin", + "type": "array", + "items": { + "$ref": "#/components/schemas/MulticastOrigin" + }, + "uniqueItems": true + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/originated_tunnel_endpoints": { "get": { "operationId": "get_originated_tunnel_endpoints", @@ -444,6 +560,104 @@ "type": "string", "pattern": "^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))\\/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8])$" }, + "MulticastOrigin": { + "description": "Origin information for a multicast group announcement.\n\nThis is analogous to TunnelOrigin but for multicast groups.\n\nThis represents a subscription to a multicast group that should be advertised via DDM. The overlay_group is the application-visible multicast address (e.g., 233.252.0.1 or ff0e::1), while underlay_group is the mapped admin-local scoped IPv6 address (ff04::X) used in the underlay network.", + "type": "object", + "properties": { + "metric": { + "description": "Metric for path selection (lower is better).\n\nUsed for multi-rack replication optimization. Excluded from identity (Hash/Eq) so that metric changes update an existing entry rather than creating a duplicate.", + "default": 0, + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "overlay_group": { + "description": "The overlay multicast group address (IPv4 or IPv6). This is the group address visible to applications.", + "type": "string", + "format": "ip" + }, + "source": { + "nullable": true, + "description": "Optional source address for Source-Specific Multicast (S,G) routes. None for Any-Source Multicast (*,G) routes.", + "default": null, + "type": "string", + "format": "ip" + }, + "underlay_group": { + "description": "The underlay multicast group address (ff04::X). Validated at construction to be within ff04::/64.", + "type": "string", + "format": "ipv6" + }, + "vni": { + "description": "VNI for this multicast group (identifies the VPC/network context).", + "default": 77, + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + "required": [ + "overlay_group", + "underlay_group" + ] + }, + "MulticastPathHop": { + "description": "A single hop in the multicast path, carrying metadata needed for replication optimization.\n\nUnlike unicast paths which only need hostnames, multicast hops carry additional information for computing optimal replication points per [RFD 488].\n\n[RFD 488]: https://rfd.shared.oxide.computer/rfd/0488", + "type": "object", + "properties": { + "downstream_subscriber_count": { + "description": "Number of downstream subscribers reachable via this hop. Used for load-aware replication decisions in multi-rack topologies.", + "default": 0, + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "router_id": { + "description": "Router identifier (hostname).", + "type": "string" + }, + "underlay_addr": { + "description": "The underlay address of this router (for replication targeting).", + "type": "string", + "format": "ipv6" + } + }, + "required": [ + "router_id", + "underlay_addr" + ] + }, + "MulticastRoute": { + "description": "A multicast route learned via DDM.\n\nCarries both the group origin and the path vector from the originating subscriber through intermediate transit routers. The path enables loop detection and (in multi-rack topologies) replication optimizations per [RFD 488] in the future.\n\nEquality and hashing consider only `origin` and `nexthop` so that a route update with a longer path replaces the existing entry in hash-based collections.\n\n[RFD 488]: https://rfd.shared.oxide.computer/rfd/0488", + "type": "object", + "properties": { + "nexthop": { + "description": "Underlay nexthop address (DDM peer that advertised this route). Used to associate the route with a peer for expiration.", + "type": "string", + "format": "ipv6" + }, + "origin": { + "description": "The multicast group origin information.", + "allOf": [ + { + "$ref": "#/components/schemas/MulticastOrigin" + } + ] + }, + "path": { + "description": "Path vector from the originating subscriber outward. Each hop records the router that redistributed this subscription announcement. Used for loop detection on pull and for future replication optimization in multi-rack topologies.", + "default": [], + "type": "array", + "items": { + "$ref": "#/components/schemas/MulticastPathHop" + } + } + }, + "required": [ + "nexthop", + "origin" + ] + }, "PathVector": { "type": "object", "properties": { diff --git a/openapi/ddm-admin/ddm-admin-latest.json b/openapi/ddm-admin/ddm-admin-latest.json index 454466590..39659731d 120000 --- a/openapi/ddm-admin/ddm-admin-latest.json +++ b/openapi/ddm-admin/ddm-admin-latest.json @@ -1 +1 @@ -ddm-admin-1.0.0-b6eac7.json \ No newline at end of file +ddm-admin-2.0.0-3dc476.json \ No newline at end of file From 86ec97f55d6fc7b7f1c087b40b9ce35a96d1e37d Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Tue, 7 Apr 2026 02:48:34 +0000 Subject: [PATCH 02/16] [review] address PR review: doc comments, lock! cleanup --- .../versions/src/multicast_support/db.rs | 19 +++-- .../src/multicast_support/exchange.rs | 20 +++--- ddm/src/exchange.rs | 11 +-- mg-lower/src/platform.rs | 69 +++++++++---------- ...dc476.json => ddm-admin-2.0.0-bdf299.json} | 4 +- openapi/ddm-admin/ddm-admin-latest.json | 2 +- ...fc8d9c.json => mg-admin-8.0.0-082a16.json} | 2 +- openapi/mg-admin/mg-admin-latest.json | 2 +- rdb-types/src/lib.rs | 4 +- 9 files changed, 64 insertions(+), 69 deletions(-) rename openapi/ddm-admin/{ddm-admin-2.0.0-3dc476.json => ddm-admin-2.0.0-bdf299.json} (96%) rename openapi/mg-admin/{mg-admin-8.0.0-fc8d9c.json => mg-admin-8.0.0-082a16.json} (99%) diff --git a/ddm-types/versions/src/multicast_support/db.rs b/ddm-types/versions/src/multicast_support/db.rs index f6cdf4d01..8fc2e4553 100644 --- a/ddm-types/versions/src/multicast_support/db.rs +++ b/ddm-types/versions/src/multicast_support/db.rs @@ -12,16 +12,15 @@ use crate::v2::exchange::MulticastPathHop; /// A multicast route learned via DDM. /// -/// Carries both the group origin and the path vector from the -/// originating subscriber through intermediate transit routers. -/// The path enables loop detection and (in multi-rack topologies) -/// replication optimizations per [RFD 488] in the future. -/// -/// Equality and hashing consider only `origin` and `nexthop` so that -/// a route update with a longer path replaces the existing entry in -/// hash-based collections. -/// -/// [RFD 488]: https://rfd.shared.oxide.computer/rfd/0488 +/// Carries a MulticastOrigin (overlay group + ff04::/64 underlay +/// mapping) and the path vector from the originating subscriber +/// through intermediate transit routers. +// The path enables loop detection and (in multi-rack topologies) +// replication optimizations (RFD 488) in the future. +// +// Equality and hashing consider only `origin` and `nexthop` so that +// a route update with a longer path replaces the existing entry in +// hash-based collections. #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] pub struct MulticastRoute { /// The multicast group origin information. diff --git a/ddm-types/versions/src/multicast_support/exchange.rs b/ddm-types/versions/src/multicast_support/exchange.rs index 4aa528ed5..ca0cb161f 100644 --- a/ddm-types/versions/src/multicast_support/exchange.rs +++ b/ddm-types/versions/src/multicast_support/exchange.rs @@ -8,12 +8,9 @@ use std::net::Ipv6Addr; /// A single hop in the multicast path, carrying metadata needed for /// replication optimization. -/// -/// Unlike unicast paths which only need hostnames, multicast hops carry -/// additional information for computing optimal replication points per -/// [RFD 488]. -/// -/// [RFD 488]: https://rfd.shared.oxide.computer/rfd/0488 +// Unlike unicast paths which only need hostnames, multicast hops carry +// additional information for computing optimal replication points +// (RFD 488). #[derive( Debug, Clone, PartialEq, Eq, Hash, Deserialize, Serialize, JsonSchema, )] @@ -46,11 +43,12 @@ impl MulticastPathHop { /// Multicast group subscription announcement propagating through DDM. /// -/// The path records the sequence of routers from the original subscriber -/// toward the current receiving router. Currently, this is used for loop -/// detection: if our router_id appears in the path, the announcement has -/// already traversed us and is dropped. The path structure also carries -/// topology information for future replication optimizations (RFD 488). +/// Contains a MulticastOrigin (overlay group + ff04::/64 underlay +/// mapping) and the path from the original subscriber outward. +// Currently, this is used for loop detection: if our router_id appears in the +// path, the announcement has already traversed us and is dropped. The path +// structure also carries topology information for future replication +// optimizations (RFD 488). #[derive( Debug, Clone, PartialEq, Eq, Hash, Deserialize, Serialize, JsonSchema, )] diff --git a/ddm/src/exchange.rs b/ddm/src/exchange.rs index 57204cc83..3bea9016b 100644 --- a/ddm/src/exchange.rs +++ b/ddm/src/exchange.rs @@ -406,8 +406,11 @@ impl TunnelUpdate { /// Multicast group subscription updates. /// -/// Carries path-vector information for multicast group subscriptions, -/// enabling loop detection and optimal replication point computation. +/// Each entry carries a [`MulticastPathVector`] containing a +/// [`MulticastOrigin`] (overlay group + ff04::/64 underlay mapping) +/// and the path vector for loop detection. +/// +/// [`MulticastOrigin`]: mg_common::net::MulticastOrigin #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] pub struct MulticastUpdate { pub announce: HashSet, @@ -1210,9 +1213,7 @@ fn handle_underlay_update(update: &UnderlayUpdate, ctx: &HandlerContext) { /// apply at this layer. The MRIB RPF module in rdb handles that check /// before routes are originated into DDM. At the DDM exchange level, /// the path vector provides loop detection and carries topology -/// information for replication optimization per [RFD 488]. -/// -/// [RFD 488]: https://rfd.shared.oxide.computer/rfd/0488 +/// information for replication optimization (RFD 488). fn handle_multicast_update(update: &MulticastUpdate, ctx: &HandlerContext) { let db = &ctx.ctx.db; let hostname = &ctx.ctx.hostname; diff --git a/mg-lower/src/platform.rs b/mg-lower/src/platform.rs index 363dceafe..1d453a1ee 100644 --- a/mg-lower/src/platform.rs +++ b/mg-lower/src/platform.rs @@ -217,6 +217,13 @@ pub trait Ddm { ddm_admin_client::Error, >; + /// Get multicast group subscriptions originated by this router. + /// + /// Each `MulticastOrigin` pairs an overlay group address with its + /// underlay mapping (ff04::/64) and optional source for (S,G) routes. + /// + /// Method names follow the DDM admin API convention + /// (`originated_multicast_groups`, not `originated_multicast_origins`). async fn get_originated_multicast_groups( &self, ) -> Result< @@ -224,6 +231,10 @@ pub trait Ddm { ddm_admin_client::Error, >; + /// Advertise multicast group subscriptions to DDM peers. + /// + /// Each entry is a `MulticastOrigin` pairing an overlay group + /// with its ff04::/64 underlay mapping. #[allow(clippy::ptr_arg)] async fn advertise_multicast_groups<'a>( &'a self, @@ -233,6 +244,10 @@ pub trait Ddm { ddm_admin_client::Error, >; + /// Withdraw multicast group subscriptions from DDM peers. + /// + /// Each entry is a `MulticastOrigin` pairing an overlay group + /// with its ff04::/64 underlay mapping. #[allow(clippy::ptr_arg)] async fn withdraw_multicast_groups<'a>( &'a self, @@ -484,6 +499,7 @@ pub(crate) mod test { use crate::MG_LOWER_TAG; use super::*; + use mg_common::lock; use std::sync::Mutex; use std::{collections::HashMap, net::IpAddr}; @@ -538,7 +554,7 @@ pub(crate) mod test { link_id: &LinkId, ) -> Result, DpdClientError> { - let links = self.links.lock().unwrap(); + let links = lock!(self.links); let link = links .iter() .find(|x| &x.port_id == port_id && &x.link_id == link_id); @@ -556,10 +572,7 @@ pub(crate) mod test { dpd_client::ResponseValue>, DpdClientError, > { - let result = self - .v4_routes - .lock() - .unwrap() + let result = lock!(self.v4_routes) .get(cidr) .cloned() .unwrap_or(Vec::default()); @@ -573,10 +586,7 @@ pub(crate) mod test { dpd_client::ResponseValue>, DpdClientError, > { - let result = self - .v6_routes - .lock() - .unwrap() + let result = lock!(self.v6_routes) .get(cidr) .cloned() .unwrap_or(Vec::default()); @@ -588,7 +598,7 @@ pub(crate) mod test { addr: &Ipv6Entry, ) -> Result, DpdClientError> { - self.loopback.lock().unwrap().replace(addr.clone()); + lock!(self.loopback).replace(addr.clone()); Ok(dpd_response_ok!(())) } @@ -599,7 +609,7 @@ pub(crate) mod test { dpd_client::ResponseValue>, DpdClientError, > { - let links = self.links.lock().unwrap(); + let links = lock!(self.links); let result = links .iter() .filter(|x| match filter { @@ -666,7 +676,7 @@ pub(crate) mod test { RouteTarget::V4(v4) => Route::V4(v4.clone()), RouteTarget::V6(v6) => Route::V6(v6.clone()), }; - let mut routes = self.v4_routes.lock().unwrap(); + let mut routes = lock!(self.v4_routes); match routes.get_mut(&body.cidr) { Some(targets) => { targets.push(route); @@ -683,7 +693,7 @@ pub(crate) mod test { body: &'a Ipv6RouteUpdate, ) -> Result, DpdClientError> { - let mut routes = self.v6_routes.lock().unwrap(); + let mut routes = lock!(self.v6_routes); match routes.get_mut(&body.cidr) { Some(targets) => { targets.push(body.target.clone()); @@ -703,7 +713,7 @@ pub(crate) mod test { tgt_ip: &'a IpAddr, ) -> Result, DpdClientError> { - let mut routes = self.v4_routes.lock().unwrap(); + let mut routes = lock!(self.v4_routes); if let Some(targets) = routes.get_mut(cidr) { targets.retain(|x| match (x, tgt_ip) { (Route::V4(x), IpAddr::V4(ip)) => { @@ -731,7 +741,7 @@ pub(crate) mod test { tgt_ip: &'a std::net::Ipv6Addr, ) -> Result, DpdClientError> { - let mut routes = self.v6_routes.lock().unwrap(); + let mut routes = lock!(self.v6_routes); if let Some(targets) = routes.get_mut(cidr) { targets.retain(|x| { !(x.tgt_ip == *tgt_ip @@ -773,9 +783,7 @@ pub(crate) mod test { ddm_admin_client::ResponseValue>, ddm_admin_client::Error, > { - Ok(ddm_response_ok!( - self.tunnel_originated.lock().unwrap().clone() - )) + Ok(ddm_response_ok!(lock!(self.tunnel_originated).clone())) } async fn get_originated( @@ -784,7 +792,7 @@ pub(crate) mod test { ddm_admin_client::ResponseValue>, ddm_admin_client::Error, > { - Ok(ddm_response_ok!(self.originated.lock().unwrap().clone())) + Ok(ddm_response_ok!(lock!(self.originated).clone())) } async fn advertise_prefixes<'a>( @@ -794,7 +802,7 @@ pub(crate) mod test { ddm_admin_client::ResponseValue<()>, ddm_admin_client::Error, > { - self.originated.lock().unwrap().extend(body); + lock!(self.originated).extend(body); Ok(ddm_response_ok!(())) } @@ -805,7 +813,7 @@ pub(crate) mod test { ddm_admin_client::ResponseValue<()>, ddm_admin_client::Error, > { - self.tunnel_originated.lock().unwrap().extend(body.clone()); + lock!(self.tunnel_originated).extend(body.clone()); Ok(ddm_response_ok!(())) } @@ -816,10 +824,7 @@ pub(crate) mod test { ddm_admin_client::ResponseValue<()>, ddm_admin_client::Error, > { - self.tunnel_originated - .lock() - .unwrap() - .retain(|x| !body.contains(x)); + lock!(self.tunnel_originated).retain(|x| !body.contains(x)); Ok(ddm_response_ok!(())) } @@ -829,9 +834,7 @@ pub(crate) mod test { ddm_admin_client::ResponseValue>, ddm_admin_client::Error, > { - Ok(ddm_response_ok!( - self.multicast_originated.lock().unwrap().clone() - )) + Ok(ddm_response_ok!(lock!(self.multicast_originated).clone())) } async fn advertise_multicast_groups<'a>( @@ -841,10 +844,7 @@ pub(crate) mod test { ddm_admin_client::ResponseValue<()>, ddm_admin_client::Error, > { - self.multicast_originated - .lock() - .unwrap() - .extend(body.clone()); + lock!(self.multicast_originated).extend(body.clone()); Ok(ddm_response_ok!(())) } @@ -855,10 +855,7 @@ pub(crate) mod test { ddm_admin_client::ResponseValue<()>, ddm_admin_client::Error, > { - self.multicast_originated - .lock() - .unwrap() - .retain(|x| !body.contains(x)); + lock!(self.multicast_originated).retain(|x| !body.contains(x)); Ok(ddm_response_ok!(())) } } diff --git a/openapi/ddm-admin/ddm-admin-2.0.0-3dc476.json b/openapi/ddm-admin/ddm-admin-2.0.0-bdf299.json similarity index 96% rename from openapi/ddm-admin/ddm-admin-2.0.0-3dc476.json rename to openapi/ddm-admin/ddm-admin-2.0.0-bdf299.json index 954fd3ad4..aad553619 100644 --- a/openapi/ddm-admin/ddm-admin-2.0.0-3dc476.json +++ b/openapi/ddm-admin/ddm-admin-2.0.0-bdf299.json @@ -602,7 +602,7 @@ ] }, "MulticastPathHop": { - "description": "A single hop in the multicast path, carrying metadata needed for replication optimization.\n\nUnlike unicast paths which only need hostnames, multicast hops carry additional information for computing optimal replication points per [RFD 488].\n\n[RFD 488]: https://rfd.shared.oxide.computer/rfd/0488", + "description": "A single hop in the multicast path, carrying metadata needed for replication optimization.", "type": "object", "properties": { "downstream_subscriber_count": { @@ -628,7 +628,7 @@ ] }, "MulticastRoute": { - "description": "A multicast route learned via DDM.\n\nCarries both the group origin and the path vector from the originating subscriber through intermediate transit routers. The path enables loop detection and (in multi-rack topologies) replication optimizations per [RFD 488] in the future.\n\nEquality and hashing consider only `origin` and `nexthop` so that a route update with a longer path replaces the existing entry in hash-based collections.\n\n[RFD 488]: https://rfd.shared.oxide.computer/rfd/0488", + "description": "A multicast route learned via DDM.\n\nCarries a MulticastOrigin (overlay group + ff04::/64 underlay mapping) and the path vector from the originating subscriber through intermediate transit routers.", "type": "object", "properties": { "nexthop": { diff --git a/openapi/ddm-admin/ddm-admin-latest.json b/openapi/ddm-admin/ddm-admin-latest.json index 39659731d..ba4534e57 120000 --- a/openapi/ddm-admin/ddm-admin-latest.json +++ b/openapi/ddm-admin/ddm-admin-latest.json @@ -1 +1 @@ -ddm-admin-2.0.0-3dc476.json \ No newline at end of file +ddm-admin-2.0.0-bdf299.json \ No newline at end of file diff --git a/openapi/mg-admin/mg-admin-8.0.0-fc8d9c.json b/openapi/mg-admin/mg-admin-8.0.0-082a16.json similarity index 99% rename from openapi/mg-admin/mg-admin-8.0.0-fc8d9c.json rename to openapi/mg-admin/mg-admin-8.0.0-082a16.json index b264b1094..77f5d427c 100644 --- a/openapi/mg-admin/mg-admin-8.0.0-fc8d9c.json +++ b/openapi/mg-admin/mg-admin-8.0.0-082a16.json @@ -5440,7 +5440,7 @@ ] }, "PeerId": { - "description": "Identifies a BGP peer for session management and route tracking.\n\nBGP peers can be identified in two ways: - **Numbered**: Traditional BGP peering using explicit IP addresses - **Unnumbered**: Modern peering using interface names with link-local addresses\n\n# Unnumbered Peering\n\nUnnumbered BGP uses interface names as stable identifiers instead of IP addresses. This is important because: - Link-local IPv6 addresses are discovered dynamically via NDP - Multiple interfaces may have peers with the same link-local address (e.g., fe80::1 on eth0 and fe80::1 on eth1) - Scope ID (interface index) disambiguates link-local addresses, but is not stable across reboots - Interface names provide stable, unambiguous peer identification\n\n# Route Tracking\n\nThis type is used in [`BgpPathProperties`](crate::BgpPathProperties) to track which peer advertised a route. Using `PeerId` instead of `IpAddr` ensures: - Unnumbered peers are properly distinguished even if they share link-local IPs - Route cleanup correctly removes only the routes from the intended peer - No cross-contamination when multiple unnumbered sessions exist\n\n# Examples\n\n``` use rdb_types::PeerId; use std::net::IpAddr;\n\n// Numbered peer let numbered = PeerId::Ip(\"192.0.2.1\".parse::().unwrap());\n\n// Unnumbered peer let unnumbered = PeerId::Interface(\"eth0\".to_string()); ```", + "description": "Identifies a BGP peer for session management and route tracking.\n\nBGP peers can be identified in two ways: - **Numbered**: Traditional BGP peering using explicit IP addresses - **Unnumbered**: Modern peering using interface names with link-local addresses\n\n# Unnumbered Peering\n\nUnnumbered BGP uses interface names as stable identifiers instead of IP addresses. This is important because: - Link-local IPv6 addresses are discovered dynamically via NDP - Multiple interfaces may have peers with the same link-local address (e.g., fe80::1 on eth0 and fe80::1 on eth1) - Scope ID (interface index) disambiguates link-local addresses, but is not stable across reboots - Interface names provide stable, unambiguous peer identification\n\n# Route Tracking\n\nThis type is used in `BgpPathProperties` to track which peer advertised a route. Using `PeerId` instead of `IpAddr` ensures: - Unnumbered peers are properly distinguished even if they share link-local IPs - Route cleanup correctly removes only the routes from the intended peer - No cross-contamination when multiple unnumbered sessions exist\n\n# Examples\n\n``` use rdb_types::PeerId; use std::net::IpAddr;\n\n// Numbered peer let numbered = PeerId::Ip(\"192.0.2.1\".parse::().unwrap());\n\n// Unnumbered peer let unnumbered = PeerId::Interface(\"eth0\".to_string()); ```", "oneOf": [ { "description": "Numbered peer identified by IP address\n\nUsed for traditional BGP sessions where peers are configured with explicit IP addresses (either IPv4 or IPv6 global unicast).", diff --git a/openapi/mg-admin/mg-admin-latest.json b/openapi/mg-admin/mg-admin-latest.json index 329966e4a..38b6e356e 120000 --- a/openapi/mg-admin/mg-admin-latest.json +++ b/openapi/mg-admin/mg-admin-latest.json @@ -1 +1 @@ -mg-admin-8.0.0-fc8d9c.json \ No newline at end of file +mg-admin-8.0.0-082a16.json \ No newline at end of file diff --git a/rdb-types/src/lib.rs b/rdb-types/src/lib.rs index af365e515..44a3777cc 100644 --- a/rdb-types/src/lib.rs +++ b/rdb-types/src/lib.rs @@ -459,8 +459,8 @@ pub enum ProtocolFilter { /// /// # Route Tracking /// -/// This type is used in [`BgpPathProperties`](crate::BgpPathProperties) to track -/// which peer advertised a route. Using `PeerId` instead of `IpAddr` ensures: +/// This type is used in `BgpPathProperties` to track which peer advertised a +/// route. Using `PeerId` instead of `IpAddr` ensures: /// - Unnumbered peers are properly distinguished even if they share link-local IPs /// - Route cleanup correctly removes only the routes from the intended peer /// - No cross-contamination when multiple unnumbered sessions exist From b5e96c8625eefecee0860491d26594d06e3a1ae8 Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Wed, 8 Apr 2026 02:33:51 +0000 Subject: [PATCH 03/16] [review+arch-change] address PR review + revisit M2P/OPTE handling This addresses review feedback on DDM multicast exchange PR (#696). Additionally, oriented towards the goal of Omicron owning all OPTE M2P (multicast-to-physical) mappings via the sled-agent, we remove direct OPTE M2P writes from DDM. This was an oversight by me using similar patterns to tunnel routing. The M2P table is global to xde, so having both DDM and Omicron's reconciler write to it creates a conflict risk where Nexus could reap DDM-written entries as stale. Our original intention has always been Omicron driving most things (-> Dpd/Dendrite, -> OPTE) due to the implicit and dynamic lifecycle of multicast groups. The work here is for DDM to distribute multicast membership and expose learned state via its admin API for Omicron to consume (and rely on for port knowledge), which is currently a TODO that will be removed in Omicron once this work is plumbed up. Other changes: - Replace String errors with UnderlayMulticastError in mg-common - Use route key VNI instead of hard-coded DEFAULT_MULTICAST_VNI - Sort ddmadm multicast output by overlay group for deterministic display - Consolidate oxide_vpc imports in sys.rs - Derive Eq on MulticastOrigin, document PartialEq/Hash exclusion of metric with #649 reference - Downgrade multicast withdrawal success log to debug - Fix MRIB diagram indentation - Remove "old" zl/mrib rdb-types UnderlayMulticastIpv6 and use mg-common one - Remove unnecessary DEFAULT_MULTICAST_VNI constant --- Cargo.lock | 3 + ddm-admin-client/Cargo.toml | 1 + ddm-admin-client/src/lib.rs | 26 +++++ ddm/src/exchange.rs | 71 ++++-------- ddm/src/lib.rs | 7 ++ ddm/src/sm.rs | 40 +++---- ddm/src/sys.rs | 95 +--------------- ddmadm/src/main.rs | 21 +++- mg-common/Cargo.toml | 1 + mg-common/src/net.rs | 53 +++++---- mg-lower/src/ddm.rs | 2 +- mg-lower/src/mrib.rs | 46 ++------ mg-types/versions/Cargo.toml | 1 + .../versions/src/multicast_support/mrib.rs | 5 +- mgadm/src/mrib.rs | 4 +- ...df299.json => ddm-admin-2.0.0-5318c9.json} | 14 ++- openapi/ddm-admin/ddm-admin-latest.json | 2 +- rdb/src/db.rs | 8 +- rdb/src/types.rs | 106 +++--------------- 19 files changed, 177 insertions(+), 329 deletions(-) rename openapi/ddm-admin/{ddm-admin-2.0.0-bdf299.json => ddm-admin-2.0.0-5318c9.json} (98%) diff --git a/Cargo.lock b/Cargo.lock index f8c9e9e89..812378285 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1349,6 +1349,7 @@ dependencies = [ name = "ddm-admin-client" version = "0.1.0" dependencies = [ + "mg-common", "oxnet", "progenitor 0.13.0", "reqwest 0.13.2", @@ -3803,6 +3804,7 @@ dependencies = [ "slog-async", "slog-bunyan", "smf 0.10.0 (git+https://github.com/illumos/smf-rs?branch=main)", + "thiserror 2.0.18", "uuid", ] @@ -3878,6 +3880,7 @@ version = "0.1.0" dependencies = [ "bfd", "bgp", + "mg-common", "rdb", "schemars 0.8.22", "serde", diff --git a/ddm-admin-client/Cargo.toml b/ddm-admin-client/Cargo.toml index 24d50602d..88e581870 100644 --- a/ddm-admin-client/Cargo.toml +++ b/ddm-admin-client/Cargo.toml @@ -7,6 +7,7 @@ edition = "2024" ignored = ["oxnet", "serde", "uuid"] [dependencies] +mg-common.workspace = true oxnet.workspace = true progenitor.workspace = true reqwest.workspace = true diff --git a/ddm-admin-client/src/lib.rs b/ddm-admin-client/src/lib.rs index 0d22065fe..bea6d0283 100644 --- a/ddm-admin-client/src/lib.rs +++ b/ddm-admin-client/src/lib.rs @@ -40,6 +40,20 @@ impl std::hash::Hash for types::TunnelOrigin { } } +impl std::cmp::PartialEq for types::Vni { + fn eq(&self, other: &Self) -> bool { + self.0.eq(&other.0) + } +} + +impl std::cmp::Eq for types::Vni {} + +impl std::hash::Hash for types::Vni { + fn hash(&self, state: &mut H) { + self.0.hash(state); + } +} + impl std::cmp::PartialEq for types::MulticastOrigin { fn eq(&self, other: &Self) -> bool { self.overlay_group.eq(&other.overlay_group) @@ -61,3 +75,15 @@ impl std::hash::Hash for types::MulticastOrigin { self.source.hash(state); } } + +impl From for types::MulticastOrigin { + fn from(o: mg_common::net::MulticastOrigin) -> Self { + Self { + overlay_group: o.overlay_group, + underlay_group: o.underlay_group.ip(), + vni: types::Vni(o.vni.as_u32()), + metric: o.metric, + source: o.source, + } + } +} diff --git a/ddm/src/exchange.rs b/ddm/src/exchange.rs index 3bea9016b..9e696e371 100644 --- a/ddm/src/exchange.rs +++ b/ddm/src/exchange.rs @@ -977,10 +977,6 @@ fn collect_multicast( Ok(multicast) } -fn opt(s: HashSet) -> Option> { - if s.is_empty() { None } else { Some(s) } -} - #[endpoint { method = GET, path = "/v3/pull" }] async fn pull_handler_v3( ctx: RequestContext>>, @@ -988,8 +984,8 @@ async fn pull_handler_v3( let ctx = ctx.context().lock().await.clone(); let (underlay, tunnel) = collect_underlay_tunnel(&ctx)?; Ok(HttpResponseOk(PullResponseV3 { - underlay: opt(underlay), - tunnel: opt(tunnel), + underlay: crate::non_empty(underlay), + tunnel: crate::non_empty(tunnel), })) } @@ -1001,9 +997,9 @@ async fn pull_handler_v4( let (underlay, tunnel) = collect_underlay_tunnel(&ctx)?; let multicast = collect_multicast(&ctx)?; Ok(HttpResponseOk(PullResponse { - underlay: opt(underlay), - tunnel: opt(tunnel), - multicast: opt(multicast), + underlay: crate::non_empty(underlay), + tunnel: crate::non_empty(tunnel), + multicast: crate::non_empty(multicast), })) } @@ -1253,48 +1249,29 @@ fn handle_multicast_update(update: &MulticastUpdate, ctx: &HandlerContext) { } // Atomic import + delete + diff under a single lock. - let (to_add, to_del) = db.update_imported_mcast(&import, &remove); - - if let Err(e) = crate::sys::add_multicast_routes( - &ctx.log, - &ctx.ctx.config.if_name, - &to_add, - ) { - err!( - ctx.log, - ctx.ctx.config.if_name, - "add multicast routes: {e}: {to_add:#?}", - ) - } - - if let Err(e) = crate::sys::remove_multicast_routes( - &ctx.log, - &ctx.ctx.config.if_name, - &to_del, - ) { - err!( - ctx.log, - ctx.ctx.config.if_name, - "remove multicast routes: {e}: {to_del:#?}", - ) - } + // + // DDM stores learned multicast state, which feeds back into Omicron, as + // the latter owns OPTE M2P programming via sled-agent (the M2P table is + // global to xde). + // Learned state is queryable via the DDM admin API (get_multicast_groups). + db.update_imported_mcast(&import, &remove); } #[cfg(test)] mod test { use super::*; use ddm_types::exchange::MulticastPathHop; - use mg_common::net::{MulticastOrigin, UnderlayMulticastIpv6}; + use mg_common::net::{MulticastOrigin, UnderlayMulticastIpv6, Vni}; use std::net::Ipv6Addr; fn sample_multicast_update() -> MulticastUpdate { let origin = MulticastOrigin { overlay_group: "233.252.0.1".parse().unwrap(), - underlay_group: UnderlayMulticastIpv6::new(Ipv6Addr::new( - 0xff04, 0, 0, 0, 0, 0, 0, 1, - )) + underlay_group: UnderlayMulticastIpv6::new( + "ff04::1".parse().unwrap(), + ) .unwrap(), - vni: 77, + vni: Vni::try_from(77u32).unwrap(), metric: 0, source: None, }; @@ -1352,11 +1329,11 @@ mod test { fn v4_pull_response_round_trips() { let origin = MulticastOrigin { overlay_group: "ff0e::1".parse().unwrap(), - underlay_group: UnderlayMulticastIpv6::new(Ipv6Addr::new( - 0xff04, 0, 0, 0, 0, 0, 0, 2, - )) + underlay_group: UnderlayMulticastIpv6::new( + "ff04::2".parse().unwrap(), + ) .unwrap(), - vni: 77, + vni: Vni::try_from(77u32).unwrap(), metric: 0, source: None, }; @@ -1378,11 +1355,11 @@ mod test { fn v4_pull_response_deserializes_as_v3() { let origin = MulticastOrigin { overlay_group: "233.252.0.1".parse().unwrap(), - underlay_group: UnderlayMulticastIpv6::new(Ipv6Addr::new( - 0xff04, 0, 0, 0, 0, 0, 0, 1, - )) + underlay_group: UnderlayMulticastIpv6::new( + "ff04::1".parse().unwrap(), + ) .unwrap(), - vni: 77, + vni: Vni::try_from(77u32).unwrap(), metric: 0, source: None, }; diff --git a/ddm/src/lib.rs b/ddm/src/lib.rs index 447109ba3..7699ce439 100644 --- a/ddm/src/lib.rs +++ b/ddm/src/lib.rs @@ -11,6 +11,13 @@ pub mod sm; pub mod sys; mod util; +/// Returns `None` if the set is empty, otherwise `Some(s)`. +pub(crate) fn non_empty( + set: std::collections::HashSet, +) -> Option> { + (!set.is_empty()).then_some(set) +} + #[macro_export] macro_rules! err { ($log:expr, $index:expr, $($args:tt)+) => { diff --git a/ddm/src/sm.rs b/ddm/src/sm.rs index 44e6c9165..604b90802 100644 --- a/ddm/src/sm.rs +++ b/ddm/src/sm.rs @@ -494,12 +494,9 @@ impl Exchange { self.ctx.event_channels.len() ); - let underlay = if to_remove.is_empty() { - None - } else { - Some(UnderlayUpdate::withdraw( - to_remove - .iter() + let underlay = crate::non_empty(to_remove).map(|set| { + UnderlayUpdate::withdraw( + set.iter() .map(|x| PathVector { destination: x.destination, path: { @@ -509,35 +506,30 @@ impl Exchange { }, }) .collect(), - )) - }; + ) + }); - let tunnel = if to_remove_tnl.is_empty() { - None - } else { - Some(TunnelUpdate::withdraw( - to_remove_tnl.iter().cloned().map(Into::into).collect(), - )) - }; + let tunnel = crate::non_empty(to_remove_tnl).map(|set| { + TunnelUpdate::withdraw( + set.iter().cloned().map(Into::into).collect(), + ) + }); - // Build multicast withdrawal with our hop info - let multicast = if to_remove_mcast.is_empty() { - None - } else { + // Build multicast withdrawal with our hop info. + let multicast = crate::non_empty(to_remove_mcast).map(|set| { let hop = MulticastPathHop::new( self.ctx.hostname.clone(), self.ctx.config.addr, ); - Some(MulticastUpdate::withdraw( - to_remove_mcast - .iter() + MulticastUpdate::withdraw( + set.iter() .map(|route| ddm_types::exchange::MulticastPathVector { origin: route.origin.clone(), path: vec![hop.clone()], }) .collect(), - )) - }; + ) + }); let push = Arc::new(Update { underlay, diff --git a/ddm/src/sys.rs b/ddm/src/sys.rs index 22b782f0a..915fc0b54 100644 --- a/ddm/src/sys.rs +++ b/ddm/src/sys.rs @@ -4,7 +4,7 @@ use crate::sm::{Config, DpdConfig}; use crate::{dbg, err, inf, wrn}; -use ddm_types::db::{MulticastRoute, TunnelRoute}; +use ddm_types::db::TunnelRoute; use dpd_client::Client; use dpd_client::ClientState; use dpd_client::types; @@ -359,99 +359,6 @@ pub fn remove_tunnel_routes( Ok(()) } -#[cfg(not(target_os = "illumos"))] -pub fn add_multicast_routes( - _log: &Logger, - _ifname: &str, - _routes: &HashSet, -) -> Result<(), String> { - todo!(); -} - -/// Update OPTE multicast-to-physical (M2P) table entries for learned -/// multicast routes. Each route's overlay group is mapped to the -/// corresponding underlay multicast address so that OPTE can direct -/// multicast traffic to the correct underlay destinations. -#[cfg(target_os = "illumos")] -pub fn add_multicast_routes( - log: &Logger, - ifname: &str, - routes: &HashSet, -) -> Result<(), String> { - use oxide_vpc::api::MulticastUnderlay; - use oxide_vpc::api::SetMcast2PhysReq; - - let hdl = OpteHdl::open().map_err(|e| e.to_string())?; - - for route in routes { - let underlay = - MulticastUnderlay::new(route.origin.underlay_group.ip().into()) - .map_err(|e| { - format!( - "invalid underlay multicast address {}: {e}", - route.origin.underlay_group, - ) - })?; - let req = SetMcast2PhysReq { - group: route.origin.overlay_group.into(), - underlay, - }; - let overlay = route.origin.overlay_group; - let underlay_addr = route.origin.underlay_group; - inf!(log, ifname, "adding M2P: {overlay:?} -> {underlay_addr}"); - if let Err(e) = hdl.set_m2p(&req) { - err!(log, ifname, "failed to set M2P route: {req:?}: {e}"); - } - } - - Ok(()) -} - -#[cfg(not(target_os = "illumos"))] -pub fn remove_multicast_routes( - _log: &Logger, - _ifname: &str, - _routes: &HashSet, -) -> Result<(), String> { - todo!() -} - -/// Remove OPTE M2P table entries for withdrawn multicast routes. -#[cfg(target_os = "illumos")] -pub fn remove_multicast_routes( - log: &Logger, - ifname: &str, - routes: &HashSet, -) -> Result<(), String> { - use oxide_vpc::api::ClearMcast2PhysReq; - use oxide_vpc::api::MulticastUnderlay; - - let hdl = OpteHdl::open().map_err(|e| e.to_string())?; - - for route in routes { - let underlay = - MulticastUnderlay::new(route.origin.underlay_group.ip().into()) - .map_err(|e| { - format!( - "invalid underlay multicast address {}: {e}", - route.origin.underlay_group, - ) - })?; - let req = ClearMcast2PhysReq { - group: route.origin.overlay_group.into(), - underlay, - }; - let overlay = route.origin.overlay_group; - let underlay_addr = route.origin.underlay_group; - inf!(log, ifname, "removing M2P: {overlay:?} -> {underlay_addr}"); - if let Err(e) = hdl.clear_m2p(&req) { - err!(log, ifname, "failed to clear M2P route: {req:?}: {e}"); - } - } - - Ok(()) -} - pub fn remove_underlay_routes( log: &Logger, ifname: &str, diff --git a/ddmadm/src/main.rs b/ddmadm/src/main.rs index 309b8a6f3..08ffe5153 100644 --- a/ddmadm/src/main.rs +++ b/ddmadm/src/main.rs @@ -279,6 +279,13 @@ async fn run() -> Result<()> { } SubCommand::MulticastImported => { let msg = client.get_multicast_groups().await?; + let mut routes: Vec<_> = msg.into_inner().into_iter().collect(); + routes.sort_by(|a, b| { + a.origin + .overlay_group + .cmp(&b.origin.overlay_group) + .then_with(|| a.origin.source.cmp(&b.origin.source)) + }); let mut tw = TabWriter::new(stdout()); writeln!( &mut tw, @@ -290,7 +297,7 @@ async fn run() -> Result<()> { "Source".dimmed(), "Path".dimmed(), )?; - for route in msg.into_inner() { + for route in &routes { let source = match &route.origin.source { Some(s) => s.to_string(), None => "(*,G)".to_string(), @@ -316,6 +323,12 @@ async fn run() -> Result<()> { } SubCommand::MulticastOriginated => { let msg = client.get_originated_multicast_groups().await?; + let mut origins: Vec<_> = msg.into_inner().into_iter().collect(); + origins.sort_by(|a, b| { + a.overlay_group + .cmp(&b.overlay_group) + .then_with(|| a.source.cmp(&b.source)) + }); let mut tw = TabWriter::new(stdout()); writeln!( &mut tw, @@ -326,7 +339,7 @@ async fn run() -> Result<()> { "Metric".dimmed(), "Source".dimmed(), )?; - for origin in msg.into_inner() { + for origin in &origins { let source = match &origin.source { Some(s) => s.to_string(), None => "(*,G)".to_string(), @@ -348,7 +361,7 @@ async fn run() -> Result<()> { .advertise_multicast_groups(&vec![types::MulticastOrigin { overlay_group: mg.overlay_group, underlay_group: mg.underlay_group, - vni: mg.vni, + vni: types::Vni(mg.vni), metric: mg.metric, source: mg.source, }]) @@ -359,7 +372,7 @@ async fn run() -> Result<()> { .withdraw_multicast_groups(&vec![types::MulticastOrigin { overlay_group: mg.overlay_group, underlay_group: mg.underlay_group, - vni: mg.vni, + vni: types::Vni(mg.vni), metric: mg.metric, source: mg.source, }]) diff --git a/mg-common/Cargo.toml b/mg-common/Cargo.toml index 0c40e3174..657d5556c 100644 --- a/mg-common/Cargo.toml +++ b/mg-common/Cargo.toml @@ -12,6 +12,7 @@ schemars.workspace = true slog.workspace = true slog-bunyan.workspace = true slog-async.workspace = true +thiserror.workspace = true oximeter-producer.workspace = true oximeter.workspace = true oxnet.workspace = true diff --git a/mg-common/src/net.rs b/mg-common/src/net.rs index bce080b29..8aaa69b7b 100644 --- a/mg-common/src/net.rs +++ b/mg-common/src/net.rs @@ -2,20 +2,35 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +// Re-export so consumers of MulticastOrigin.vni don't need a direct +// omicron_common dependency. +pub use omicron_common::api::external::Vni; + use omicron_common::address::UNDERLAY_MULTICAST_SUBNET; -use omicron_common::api::external::Vni; use oxnet::{IpNet, Ipv4Net, Ipv6Net}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::fmt; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use std::str::FromStr; +use thiserror::Error; -/// Default VNI for multicast routing. -pub const DEFAULT_MULTICAST_VNI: u32 = Vni::DEFAULT_MULTICAST_VNI.as_u32(); +fn default_multicast_vni() -> Vni { + Vni::DEFAULT_MULTICAST_VNI +} -fn default_multicast_vni() -> u32 { - DEFAULT_MULTICAST_VNI +/// Error constructing an [`UnderlayMulticastIpv6`] address. +#[derive(Debug, Clone, Error)] +pub enum UnderlayMulticastError { + /// The address is not within the underlay multicast subnet (ff04::/64). + #[error( + "underlay address {addr} is not within {UNDERLAY_MULTICAST_SUBNET}" + )] + NotInSubnet { addr: Ipv6Addr }, + + /// The string could not be parsed as an IPv6 address. + #[error("invalid IPv6 address: {0}")] + InvalidIpv6(#[from] std::net::AddrParseError), } /// A validated underlay multicast IPv6 address within ff04::/64. @@ -45,13 +60,11 @@ impl UnderlayMulticastIpv6 { /// /// # Errors /// - /// Returns an error if the address is not within ff04::/64. - pub fn new(value: Ipv6Addr) -> Result { + /// Returns [`UnderlayMulticastError::NotInSubnet`] if the address is + /// not within ff04::/64. + pub fn new(value: Ipv6Addr) -> Result { if !UNDERLAY_MULTICAST_SUBNET.contains(value) { - return Err(format!( - "underlay address {value} is not within \ - {UNDERLAY_MULTICAST_SUBNET}" - )); + return Err(UnderlayMulticastError::NotInSubnet { addr: value }); } Ok(Self(value)) } @@ -70,7 +83,7 @@ impl fmt::Display for UnderlayMulticastIpv6 { } impl TryFrom for UnderlayMulticastIpv6 { - type Error = String; + type Error = UnderlayMulticastError; fn try_from(value: Ipv6Addr) -> Result { Self::new(value) @@ -90,11 +103,10 @@ impl From for IpAddr { } impl FromStr for UnderlayMulticastIpv6 { - type Err = String; + type Err = UnderlayMulticastError; fn from_str(s: &str) -> Result { - let addr: Ipv6Addr = - s.parse().map_err(|e| format!("invalid IPv6: {e}"))?; + let addr: Ipv6Addr = s.parse()?; Self::new(addr) } } @@ -194,7 +206,7 @@ pub enum IpPrefix { /// advertised via DDM. The overlay_group is the application-visible multicast /// address (e.g., 233.252.0.1 or ff0e::1), while underlay_group is the mapped /// admin-local scoped IPv6 address (ff04::X) used in the underlay network. -#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[derive(Debug, Clone, Eq, Serialize, Deserialize, JsonSchema)] pub struct MulticastOrigin { /// The overlay multicast group address (IPv4 or IPv6). /// This is the group address visible to applications. @@ -206,7 +218,7 @@ pub struct MulticastOrigin { /// VNI for this multicast group (identifies the VPC/network context). #[serde(default = "default_multicast_vni")] - pub vni: u32, + pub vni: Vni, /// Metric for path selection (lower is better). /// @@ -222,6 +234,11 @@ pub struct MulticastOrigin { pub source: Option, } +// Equality and hashing consider only the identity fields (overlay_group, +// underlay_group, vni, source), not metric. This allows metric updates to +// replace existing entries in HashSet-based collections without creating +// duplicates. This type is not used in ordered collections (BTreeSet). +// See #649 for why adding Ord here would require more care. impl PartialEq for MulticastOrigin { fn eq(&self, other: &Self) -> bool { self.overlay_group == other.overlay_group @@ -231,8 +248,6 @@ impl PartialEq for MulticastOrigin { } } -impl Eq for MulticastOrigin {} - impl std::hash::Hash for MulticastOrigin { fn hash(&self, state: &mut H) { self.overlay_group.hash(state); diff --git a/mg-lower/src/ddm.rs b/mg-lower/src/ddm.rs index 45a530233..2f987477a 100644 --- a/mg-lower/src/ddm.rs +++ b/mg-lower/src/ddm.rs @@ -159,7 +159,7 @@ pub(crate) fn remove_multicast_routes< "groups" => format!("{routes:#?}") ), Ok(_) => ddm_log!(log, - info, + debug, "withdrew multicast groups"; "groups" => format!("{routes:#?}") ), diff --git a/mg-lower/src/mrib.rs b/mg-lower/src/mrib.rs index ea8069e92..8645bc8ed 100644 --- a/mg-lower/src/mrib.rs +++ b/mg-lower/src/mrib.rs @@ -10,7 +10,7 @@ //! ## Data Flow //! //! ```text -//! MRIB (loc_mrib changes) +//! MRIB (loc_mrib changes) //! | //! v [MribChangeNotification] //! mg-lower/mrib.rs @@ -27,15 +27,11 @@ use crate::ddm::{ }; use crate::platform::{Ddm, ProductionDdm}; use ddm_admin_client::types::MulticastOrigin; -use mg_common::net::DEFAULT_MULTICAST_VNI; +use mg_common::net::Vni; use rdb::Mrib; -use rdb::types::{ - DEFAULT_MULTICAST_VNI as DEFAULT_MCAST_VNI, MribChangeNotification, - MulticastAddr, MulticastRoute, -}; +use rdb::types::{MribChangeNotification, MulticastAddr, MulticastRoute}; use slog::{Logger, debug, error, info}; use std::collections::HashSet; -use std::net::IpAddr; use std::sync::Arc; use std::sync::mpsc::{RecvTimeoutError, channel}; use std::thread::sleep; @@ -43,33 +39,11 @@ use std::time::Duration; const MG_LOWER_MRIB_TAG: &str = "mg-lower-mrib"; -/// Convert an MRIB MulticastRoute to a DDM MulticastOrigin. +/// Convert an MRIB [`MulticastRoute`] to a DDM [`MulticastOrigin`]. /// -/// The MulticastOrigin captures the essential information needed for DDM -/// to advertise the multicast group subscription to other routers: -/// - overlay_group: The multicast group address (e.g., 233.252.0.1 or ff0e::1) -/// - underlay_group: The mapped underlay address within the ff04::/64 subnet -/// (a /64 within admin-local scope per RFC 7346, reserved for the rack's -/// underlay multicast traffic) -/// - source: Optional source for (S,G) routes, None for (*,G) -/// - vni: Virtual Network Identifier (default multicast VNI) -fn mrib_route_to_ddm_origin(route: &MulticastRoute) -> MulticastOrigin { - // Extract overlay group address from the route key - let overlay_group: IpAddr = match route.key.group() { - MulticastAddr::V4(v4) => IpAddr::V4(v4.ip()), - MulticastAddr::V6(v6) => IpAddr::V6(v6.ip()), - }; - - // Extract source address for (S,G) routes - let source = route.key.source(); - - MulticastOrigin { - overlay_group, - underlay_group: route.underlay_group.into(), - vni: DEFAULT_MULTICAST_VNI, - metric: 0, // Default metric - source, - } +/// [`MulticastOrigin`]: ddm_admin_client::types::MulticastOrigin +fn ddm_origin(route: &MulticastRoute) -> MulticastOrigin { + mg_common::net::MulticastOrigin::from(route).into() } /// Run the MRIB synchronization loop. @@ -135,7 +109,7 @@ pub(crate) fn full_sync( // Convert to DDM MulticastOrigin set let mrib_origins: HashSet = - mrib_routes.values().map(mrib_route_to_ddm_origin).collect(); + mrib_routes.values().map(ddm_origin).collect(); // Get current DDM advertised state let ddm_current: HashSet = rt @@ -192,7 +166,7 @@ fn handle_change( for key in notification.changed { // Check if route exists in loc_mrib (installed) if let Some(route) = mrib.get_selected_route(&key) { - let origin = mrib_route_to_ddm_origin(&route); + let origin = ddm_origin(&route); if !ddm_current.contains(&origin) { to_add.push(origin); } @@ -206,7 +180,7 @@ fn handle_change( && let Ok(ddm_key) = rdb::types::MulticastRouteKey::new( ddm_origin.source, overlay_group, - DEFAULT_MCAST_VNI, + Vni::DEFAULT_MULTICAST_VNI, ) && ddm_key == key { diff --git a/mg-types/versions/Cargo.toml b/mg-types/versions/Cargo.toml index 854e12c12..254b82e9a 100644 --- a/mg-types/versions/Cargo.toml +++ b/mg-types/versions/Cargo.toml @@ -6,6 +6,7 @@ edition = "2024" [dependencies] bfd.workspace = true bgp.workspace = true +mg-common.workspace = true rdb.workspace = true schemars.workspace = true serde.workspace = true diff --git a/mg-types/versions/src/multicast_support/mrib.rs b/mg-types/versions/src/multicast_support/mrib.rs index e6779b3ad..7e4e4f885 100644 --- a/mg-types/versions/src/multicast_support/mrib.rs +++ b/mg-types/versions/src/multicast_support/mrib.rs @@ -8,9 +8,8 @@ use std::net::IpAddr; -use rdb::types::{ - AddressFamily, MulticastRouteKey, UnderlayMulticastIpv6, Vni, -}; +use mg_common::net::UnderlayMulticastIpv6; +use rdb::types::{AddressFamily, MulticastRouteKey, Vni}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; diff --git a/mgadm/src/mrib.rs b/mgadm/src/mrib.rs index 402421d27..ab4548c55 100644 --- a/mgadm/src/mrib.rs +++ b/mgadm/src/mrib.rs @@ -21,9 +21,9 @@ use mg_admin_client::types::{ MribRpfRebuildIntervalRequest, MulticastRoute, MulticastRouteKey, RouteOriginFilter, Vni, }; -use rdb::types::{AddressFamily, DEFAULT_MULTICAST_VNI}; +use rdb::types::AddressFamily; -const DEFAULT_VNI: u32 = DEFAULT_MULTICAST_VNI.as_u32(); +const DEFAULT_VNI: u32 = rdb::Vni::DEFAULT_MULTICAST_VNI.as_u32(); fn parse_route_origin(s: &str) -> Result { match s.to_lowercase().as_str() { diff --git a/openapi/ddm-admin/ddm-admin-2.0.0-bdf299.json b/openapi/ddm-admin/ddm-admin-2.0.0-5318c9.json similarity index 98% rename from openapi/ddm-admin/ddm-admin-2.0.0-bdf299.json rename to openapi/ddm-admin/ddm-admin-2.0.0-5318c9.json index aad553619..4f9ddae9b 100644 --- a/openapi/ddm-admin/ddm-admin-2.0.0-bdf299.json +++ b/openapi/ddm-admin/ddm-admin-2.0.0-5318c9.json @@ -591,9 +591,11 @@ "vni": { "description": "VNI for this multicast group (identifies the VPC/network context).", "default": 77, - "type": "integer", - "format": "uint32", - "minimum": 0 + "allOf": [ + { + "$ref": "#/components/schemas/Vni" + } + ] } }, "required": [ @@ -758,6 +760,12 @@ "nexthop", "origin" ] + }, + "Vni": { + "description": "A Geneve Virtual Network Identifier", + "type": "integer", + "format": "uint32", + "minimum": 0 } }, "responses": { diff --git a/openapi/ddm-admin/ddm-admin-latest.json b/openapi/ddm-admin/ddm-admin-latest.json index ba4534e57..a2528890d 120000 --- a/openapi/ddm-admin/ddm-admin-latest.json +++ b/openapi/ddm-admin/ddm-admin-latest.json @@ -1 +1 @@ -ddm-admin-2.0.0-bdf299.json \ No newline at end of file +ddm-admin-2.0.0-5318c9.json \ No newline at end of file diff --git a/rdb/src/db.rs b/rdb/src/db.rs index e4e52f036..15dfa39eb 100644 --- a/rdb/src/db.rs +++ b/rdb/src/db.rs @@ -2020,14 +2020,14 @@ impl Reaper { #[cfg(test)] mod test { use crate::{ - AddressFamily, DEFAULT_MULTICAST_VNI, DEFAULT_RIB_PRIORITY_STATIC, - Path, Prefix, Prefix4, Prefix6, StaticRouteKey, + AddressFamily, DEFAULT_RIB_PRIORITY_STATIC, Path, Prefix, Prefix4, + Prefix6, StaticRouteKey, db::Db, test::{TEST_WAIT_ITERATIONS, TestDb}, types::{ MulticastAddr, MulticastAddrV4, MulticastAddrV6, MulticastRoute, MulticastRouteKey, MulticastSourceProtocol, PrefixDbKey, - UnderlayMulticastIpv6, UnicastAddrV4, UnicastAddrV6, + UnderlayMulticastIpv6, UnicastAddrV4, UnicastAddrV6, Vni, test_helpers::path_vecs_equal, }, }; @@ -2385,7 +2385,7 @@ mod test { let key = MulticastRouteKey::new( Some(s_ip), group, - DEFAULT_MULTICAST_VNI, + Vni::DEFAULT_MULTICAST_VNI, ) .expect("AF match"); let route = MulticastRoute::new( diff --git a/rdb/src/types.rs b/rdb/src/types.rs index 3a68b5624..c72559359 100644 --- a/rdb/src/types.rs +++ b/rdb/src/types.rs @@ -794,9 +794,6 @@ impl Display for PrefixChangeNotification { // MRIB (Multicast RIB) Types // ============================================================================ -/// Default VNI for fleet-wide multicast routing. -pub const DEFAULT_MULTICAST_VNI: Vni = Vni::DEFAULT_MULTICAST_VNI; - /// A validated IPv4 unicast address suitable for multicast source fields. /// /// This rejects addresses that cannot appear as a forwarded unicast source: @@ -1112,92 +1109,7 @@ impl From for Ipv6Addr { } } -/// A validated underlay multicast IPv6 address within ff04::/64. -/// -/// The Oxide rack maps overlay multicast groups 1:1 to admin-local scoped -/// IPv6 multicast addresses in `UNDERLAY_MULTICAST_SUBNET` (ff04::/64). -/// This type enforces that invariant at construction time. -/// -// TODO: This duplicates `dpd_types::mcast::UnderlayMulticastIpv6` in dendrite. -// Both should be consolidated into `omicron_common` so maghemite, dendrite, -// and omicron share a single definition. -#[derive( - Debug, - Copy, - Clone, - Eq, - PartialEq, - PartialOrd, - Ord, - Hash, - Serialize, - Deserialize, - JsonSchema, -)] -#[serde(try_from = "Ipv6Addr", into = "Ipv6Addr")] -#[schemars(transparent)] -pub struct UnderlayMulticastIpv6(Ipv6Addr); - -impl UnderlayMulticastIpv6 { - /// Create a new validated underlay multicast address. - /// - /// # Errors - /// - /// Returns an error if the address is not within `UNDERLAY_MULTICAST_SUBNET` - /// (ff04::/64). - pub fn new(value: Ipv6Addr) -> Result { - if !UNDERLAY_MULTICAST_SUBNET.contains(value) { - return Err(Error::Validation(format!( - "underlay address {value} is not within \ - {UNDERLAY_MULTICAST_SUBNET}" - ))); - } - Ok(Self(value)) - } - - /// Returns the underlying IPv6 address. - #[inline] - pub const fn ip(&self) -> Ipv6Addr { - self.0 - } -} - -impl fmt::Display for UnderlayMulticastIpv6 { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.0) - } -} - -impl TryFrom for UnderlayMulticastIpv6 { - type Error = Error; - - fn try_from(value: Ipv6Addr) -> Result { - Self::new(value) - } -} - -impl From for Ipv6Addr { - fn from(addr: UnderlayMulticastIpv6) -> Self { - addr.0 - } -} - -impl From for IpAddr { - fn from(addr: UnderlayMulticastIpv6) -> Self { - IpAddr::V6(addr.0) - } -} - -impl FromStr for UnderlayMulticastIpv6 { - type Err = Error; - - fn from_str(s: &str) -> Result { - let addr: Ipv6Addr = s.parse().map_err(|_| { - Error::Validation(format!("invalid IPv6 address: {s}")) - })?; - Self::new(addr) - } -} +pub use mg_common::net::UnderlayMulticastIpv6; /// A validated multicast group address (IPv4 or IPv6). /// @@ -1694,6 +1606,18 @@ impl MulticastRoute { } } +impl From<&MulticastRoute> for mg_common::net::MulticastOrigin { + fn from(route: &MulticastRoute) -> Self { + Self { + overlay_group: route.key.group().ip(), + underlay_group: route.underlay_group, + vni: route.key.vni(), + metric: 0, + source: route.key.source(), + } + } +} + /// Source of a multicast route entry. #[derive( Debug, Copy, Clone, Serialize, Deserialize, JsonSchema, Eq, PartialEq, @@ -2034,7 +1958,7 @@ mod test { let result = MulticastRouteKey::new( Some(IpAddr::V4(src.ip())), group.into(), - DEFAULT_MULTICAST_VNI, + Vni::DEFAULT_MULTICAST_VNI, ); assert!( result.is_err(), @@ -2049,7 +1973,7 @@ mod test { let result = MulticastRouteKey::new( Some(IpAddr::V6(src)), group.into(), - DEFAULT_MULTICAST_VNI, + Vni::DEFAULT_MULTICAST_VNI, ); assert!( result.is_err(), From bf2a7030fbaac0a0baee42e82d12fd4697dfaba4 Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Wed, 8 Apr 2026 07:28:30 +0000 Subject: [PATCH 04/16] [api] add if_name to PeerInfo for sled-to-port mapping Add an optional `if_name` field to PeerInfo (v2) so Omicron can learn which switch port a DDM peer was discovered on. This enables Omicron to use DDM as the primary source of truth for sled-to-port mapping, replacing or cross-validating the current inventory-based approach. The get_peers endpoint is versioned: v2+ returns PeerInfo with if_name, v1 returns the original PeerInfo without it. --- Cargo.lock | 1 + ddm-api/src/lib.rs | 17 ++++++++++- ddm-types/versions/src/latest.rs | 2 +- .../versions/src/multicast_support/db.rs | 28 +++++++++++++++++++ ddm/Cargo.toml | 1 + ddm/src/admin.rs | 16 +++++++++++ ddm/src/discovery.rs | 1 + ...318c9.json => ddm-admin-2.0.0-8aeda2.json} | 7 +++++ openapi/ddm-admin/ddm-admin-latest.json | 2 +- 9 files changed, 72 insertions(+), 3 deletions(-) rename openapi/ddm-admin/{ddm-admin-2.0.0-5318c9.json => ddm-admin-2.0.0-8aeda2.json} (98%) diff --git a/Cargo.lock b/Cargo.lock index 812378285..6bf266597 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1318,6 +1318,7 @@ dependencies = [ "chrono", "ddm-api", "ddm-types", + "ddm-types-versions", "dpd-client", "dropshot 0.17.0", "hostname 0.4.2", diff --git a/ddm-api/src/lib.rs b/ddm-api/src/lib.rs index 905942b59..6f3f82825 100644 --- a/ddm-api/src/lib.rs +++ b/ddm-api/src/lib.rs @@ -3,6 +3,7 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use ddm_types_versions::latest; +use ddm_types_versions::v1; use dropshot::HttpError; use dropshot::HttpResponseOk; use dropshot::HttpResponseUpdatedNoContent; @@ -46,11 +47,25 @@ api_versions!([ pub trait DdmAdminApi { type Context; - #[endpoint { method = GET, path = "/peers" }] + #[endpoint { + method = GET, + path = "/peers", + versions = VERSION_MULTICAST_SUPPORT.. + }] async fn get_peers( ctx: RequestContext, ) -> Result>, HttpError>; + /// Returns peers without interface name information. + #[endpoint { + method = GET, + path = "/peers", + versions = ..VERSION_MULTICAST_SUPPORT + }] + async fn get_peers_v1( + ctx: RequestContext, + ) -> Result>, HttpError>; + #[endpoint { method = DELETE, path = "/peers/{addr}" }] async fn expire_peer( ctx: RequestContext, diff --git a/ddm-types/versions/src/latest.rs b/ddm-types/versions/src/latest.rs index c2d29e3e0..721f8cbf8 100644 --- a/ddm-types/versions/src/latest.rs +++ b/ddm-types/versions/src/latest.rs @@ -11,11 +11,11 @@ pub mod admin { } pub mod db { - pub use crate::v1::db::PeerInfo; pub use crate::v1::db::PeerStatus; pub use crate::v1::db::RouterKind; pub use crate::v1::db::TunnelRoute; pub use crate::v2::db::MulticastRoute; + pub use crate::v2::db::PeerInfo; } pub mod exchange { diff --git a/ddm-types/versions/src/multicast_support/db.rs b/ddm-types/versions/src/multicast_support/db.rs index 8fc2e4553..8fedfe643 100644 --- a/ddm-types/versions/src/multicast_support/db.rs +++ b/ddm-types/versions/src/multicast_support/db.rs @@ -8,6 +8,7 @@ use mg_common::net::MulticastOrigin; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; +use crate::v1::db::{PeerStatus, RouterKind}; use crate::v2::exchange::MulticastPathHop; /// A multicast route learned via DDM. @@ -59,3 +60,30 @@ impl From for MulticastOrigin { x.origin } } + +/// Peer information with an optional interface name. +/// +// Adds the `if_name` field to identify which underlay interface the peer +// was discovered on. +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq)] +pub struct PeerInfo { + pub status: PeerStatus, + pub addr: Ipv6Addr, + pub host: String, + pub kind: RouterKind, + /// Interface name the peer was discovered on (e.g., "tfportrear0_0"). + #[serde(default)] + pub if_name: Option, +} + +/// Downconvert v2 PeerInfo to v1 PeerInfo by dropping `if_name`. +impl From for crate::v1::db::PeerInfo { + fn from(p: PeerInfo) -> Self { + Self { + status: p.status, + addr: p.addr, + host: p.host, + kind: p.kind, + } + } +} diff --git a/ddm/Cargo.toml b/ddm/Cargo.toml index f505632d0..7856f5a78 100644 --- a/ddm/Cargo.toml +++ b/ddm/Cargo.toml @@ -35,3 +35,4 @@ oxnet.workspace = true uuid.workspace = true ddm-api.workspace = true ddm-types.workspace = true +ddm-types-versions.workspace = true diff --git a/ddm/src/admin.rs b/ddm/src/admin.rs index 51a50677a..26fe15e34 100644 --- a/ddm/src/admin.rs +++ b/ddm/src/admin.rs @@ -116,6 +116,22 @@ impl DdmAdminApi for DdmAdminApiImpl { Ok(HttpResponseOk(ctx.db.peers())) } + async fn get_peers_v1( + ctx: RequestContext, + ) -> Result< + HttpResponseOk>, + HttpError, + > { + let ctx = lock!(ctx.context()); + let peers = ctx + .db + .peers() + .into_iter() + .map(|(k, v)| (k, v.into())) + .collect(); + Ok(HttpResponseOk(peers)) + } + async fn expire_peer( ctx: RequestContext, params: Path, diff --git a/ddm/src/discovery.rs b/ddm/src/discovery.rs index dd6da9346..2c94a08f2 100644 --- a/ddm/src/discovery.rs +++ b/ddm/src/discovery.rs @@ -527,6 +527,7 @@ fn handle_advertisement( addr: *sender, host: hostname, kind, + if_name: Some(ctx.config.if_name.clone()), }, ); if updated { diff --git a/openapi/ddm-admin/ddm-admin-2.0.0-5318c9.json b/openapi/ddm-admin/ddm-admin-2.0.0-8aeda2.json similarity index 98% rename from openapi/ddm-admin/ddm-admin-2.0.0-5318c9.json rename to openapi/ddm-admin/ddm-admin-2.0.0-8aeda2.json index 4f9ddae9b..312fadd40 100644 --- a/openapi/ddm-admin/ddm-admin-2.0.0-5318c9.json +++ b/openapi/ddm-admin/ddm-admin-2.0.0-8aeda2.json @@ -679,6 +679,7 @@ ] }, "PeerInfo": { + "description": "Peer information with an optional interface name.", "type": "object", "properties": { "addr": { @@ -688,6 +689,12 @@ "host": { "type": "string" }, + "if_name": { + "nullable": true, + "description": "Interface name the peer was discovered on (e.g., \"tfportrear0_0\").", + "default": null, + "type": "string" + }, "kind": { "$ref": "#/components/schemas/RouterKind" }, diff --git a/openapi/ddm-admin/ddm-admin-latest.json b/openapi/ddm-admin/ddm-admin-latest.json index a2528890d..aaa8691d3 120000 --- a/openapi/ddm-admin/ddm-admin-latest.json +++ b/openapi/ddm-admin/ddm-admin-latest.json @@ -1 +1 @@ -ddm-admin-2.0.0-5318c9.json \ No newline at end of file +ddm-admin-2.0.0-8aeda2.json \ No newline at end of file From a3df7a02d1ba5520115f421690a602ebb6d6887a Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Fri, 17 Apr 2026 01:58:07 +0000 Subject: [PATCH 05/16] [multicast|deps] remove mg-common dep from ddm-admin-client This moves `MulticastOrigin` conversion inline to mg-lower, which already depends on both mg-common and ddm-admin-client. This keeps ddm-admin-client lean and avoids pulling oximeter-producer and omicron-common transitively into client consumers. --- Cargo.lock | 1 - ddm-admin-client/Cargo.toml | 1 - ddm-admin-client/src/lib.rs | 12 ------------ mg-lower/src/mrib.rs | 8 +++++++- 4 files changed, 7 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3c7205894..5e73293f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1350,7 +1350,6 @@ dependencies = [ name = "ddm-admin-client" version = "0.1.0" dependencies = [ - "mg-common", "oxnet", "progenitor 0.13.0", "reqwest 0.13.2", diff --git a/ddm-admin-client/Cargo.toml b/ddm-admin-client/Cargo.toml index 88e581870..24d50602d 100644 --- a/ddm-admin-client/Cargo.toml +++ b/ddm-admin-client/Cargo.toml @@ -7,7 +7,6 @@ edition = "2024" ignored = ["oxnet", "serde", "uuid"] [dependencies] -mg-common.workspace = true oxnet.workspace = true progenitor.workspace = true reqwest.workspace = true diff --git a/ddm-admin-client/src/lib.rs b/ddm-admin-client/src/lib.rs index bea6d0283..2f65f6c9b 100644 --- a/ddm-admin-client/src/lib.rs +++ b/ddm-admin-client/src/lib.rs @@ -75,15 +75,3 @@ impl std::hash::Hash for types::MulticastOrigin { self.source.hash(state); } } - -impl From for types::MulticastOrigin { - fn from(o: mg_common::net::MulticastOrigin) -> Self { - Self { - overlay_group: o.overlay_group, - underlay_group: o.underlay_group.ip(), - vni: types::Vni(o.vni.as_u32()), - metric: o.metric, - source: o.source, - } - } -} diff --git a/mg-lower/src/mrib.rs b/mg-lower/src/mrib.rs index 8645bc8ed..803587f07 100644 --- a/mg-lower/src/mrib.rs +++ b/mg-lower/src/mrib.rs @@ -43,7 +43,13 @@ const MG_LOWER_MRIB_TAG: &str = "mg-lower-mrib"; /// /// [`MulticastOrigin`]: ddm_admin_client::types::MulticastOrigin fn ddm_origin(route: &MulticastRoute) -> MulticastOrigin { - mg_common::net::MulticastOrigin::from(route).into() + MulticastOrigin { + overlay_group: route.key.group().ip(), + underlay_group: route.underlay_group.ip(), + vni: ddm_admin_client::types::Vni(route.key.vni().as_u32()), + metric: 0, + source: route.key.source(), + } } /// Run the MRIB synchronization loop. From bc7498061b04c8e5ff670adaee73f0b46ec02b20 Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Tue, 21 Apr 2026 10:04:47 +0000 Subject: [PATCH 06/16] [mrib] sync rpf revalidation on unicast route removal --- rdb/src/db.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/rdb/src/db.rs b/rdb/src/db.rs index 06bbdc9e2..f61de0963 100644 --- a/rdb/src/db.rs +++ b/rdb/src/db.rs @@ -1452,6 +1452,19 @@ impl Db { }); } + // Synchronously revalidate affected (S,G) routes against the + // updated unicast RIB. The poptrie rebuild triggered above is + // async, so without this the MRIB update would depend on + // the rebuild thread completing first. The linear-scan fallback + // in rpf_table's lookup is sufficient here. + for prefix in &pcn.changed { + let event = match prefix { + Prefix::V4(p) => crate::mrib::rpf::RebuildEvent::V4(Some(*p)), + Prefix::V6(p) => crate::mrib::rpf::RebuildEvent::V6(Some(*p)), + }; + self.revalidate_mrib(Some(event)); + } + self.notify(pcn); Ok(()) } From 8949e8263d1e95465b87ea95314a16dcd1af24f6 Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Thu, 7 May 2026 00:54:52 +0000 Subject: [PATCH 07/16] [ddmd] add --no-state-machine flag for test fixtures and Linux build Omicron's oxidecomputer/omicron#10381 introduces a stubbed `ddmd` admin endpoint because spawning a real `ddmd` in a generic test toolchain is not viable: the routing state machine (discovery, exchange, route synchronization) depends on illumos networking facilities the toolchain does not provide. Consumers of the stub, e.g., Nexus RPW (multicast members), sled-agent's DDM reconciler, and anything that resolves the DDM internal-DNS service name, cannot exercise the real admin surface from Omicron's test harness. This work adds an opt-in `--no-state-machine` flag to `ddmd` that runs only the admin API server and skips the state machine entirely, allowing the fixture to spawn the real binary. This is analogous to `mgd --no-bgp-dispatcher`, which Omicron's `MgdInstance` already uses for the same purpose. To make the fixture path usable on Linux, `ddmd` itself must build on Linux. The previous code pulled the illumos-only crates `libnet`, `dpd-client`, `opte-ioctl`, and `oxide-vpc` unconditionally through `ddm`, which failed to link on Linux (`-lzfs`, `-ldlpi`). This change introduces an `illumos` feature in both `ddm` and `ddmd` (default-on, mirroring `mgd`'s `mg-lower` pattern) that marks those four crates optional. The buildomat `linux.sh` job now builds `ddmd` and `ddmadm`, with `ddmd` invoked as `cargo build --bin ddmd --no-default-features`. The illumos-only halves of `ddm` are isolated by the feature gate: - The routing state machine implementation moves from `sm.rs` into `sm/state.rs`. - The exchange runtime (HTTP push/pull and route programming) moves from `exchange.rs` into `exchange/runtime.rs`. - The discovery runtime (UDPv6 solicitation/advertisement loops) moves from `discovery.rs` into `discovery/runtime.rs`. Each parent `mod.rs` keeps the platform-agnostic types and re-exports the runtime surface so existing call sites resolve unchanged on illumos. The runtime submodules are gated as a unit by `#[cfg(all(feature = "illumos", target_os = "illumos"))]`. We also remove the single-function `ddm/src/util.rs`, inlining the function into `discovery/runtime.rs`, where its sole caller lives. The SIGTERM cleanup handler is installed regardless of the flag, so Ctrl-C still exits cleanly in `--no-state-machine` mode. The imported route sets are empty in that mode, so the cleanup itself is a noop. Passing `--addr` alongside `--no-state-machine` is harmless but ignored, with a warning logged. --- .github/buildomat/jobs/linux.sh | 38 ++ ddm/Cargo.toml | 16 +- ddm/src/admin.rs | 16 +- ddm/src/discovery/mod.rs | 118 ++++ .../{discovery.rs => discovery/runtime.rs} | 126 +--- ddm/src/exchange/mod.rs | 585 +++++++++++++++++ ddm/src/{exchange.rs => exchange/runtime.rs} | 597 +----------------- ddm/src/lib.rs | 7 +- ddm/src/sm/mod.rs | 195 ++++++ ddm/src/{sm.rs => sm/state.rs} | 205 +----- ddm/src/util.rs | 14 - ddmd/Cargo.toml | 6 +- ddmd/src/main.rs | 206 ++++-- 13 files changed, 1178 insertions(+), 951 deletions(-) create mode 100644 ddm/src/discovery/mod.rs rename ddm/src/{discovery.rs => discovery/runtime.rs} (73%) create mode 100644 ddm/src/exchange/mod.rs rename ddm/src/{exchange.rs => exchange/runtime.rs} (58%) create mode 100644 ddm/src/sm/mod.rs rename ddm/src/{sm.rs => sm/state.rs} (88%) delete mode 100644 ddm/src/util.rs diff --git a/.github/buildomat/jobs/linux.sh b/.github/buildomat/jobs/linux.sh index 8fc9e7eea..5a21404a1 100755 --- a/.github/buildomat/jobs/linux.sh +++ b/.github/buildomat/jobs/linux.sh @@ -28,6 +28,26 @@ #: series = "linux" #: name = "mgadm.sha256.txt" #: from_output = "/work/release/mgadm.sha256.txt" +#: +#: [[publish]] +#: series = "linux" +#: name = "ddmd" +#: from_output = "/work/release/ddmd" +#: +#: [[publish]] +#: series = "linux" +#: name = "ddmd.sha256.txt" +#: from_output = "/work/release/ddmd.sha256.txt" +#: +#: [[publish]] +#: series = "linux" +#: name = "ddmadm" +#: from_output = "/work/release/ddmadm" +#: +#: [[publish]] +#: series = "linux" +#: name = "ddmadm.sha256.txt" +#: from_output = "/work/release/ddmadm.sha256.txt" set -o errexit set -o pipefail @@ -64,3 +84,21 @@ popd cp target/debug/mgadm /work/debug cp target/release/mgadm /work/release digest /work/release/mgadm > /work/release/mgadm.sha256.txt + +banner "ddmd" +pushd ddmd +cargo build --bin ddmd --no-default-features +cargo build --bin ddmd --no-default-features --release +popd +cp target/debug/ddmd /work/debug +cp target/release/ddmd /work/release +digest /work/release/ddmd > /work/release/ddmd.sha256.txt + +banner "ddmadm" +pushd ddmadm +cargo build --bin ddmadm +cargo build --bin ddmadm --release +popd +cp target/debug/ddmadm /work/debug +cp target/release/ddmadm /work/release +digest /work/release/ddmadm > /work/release/ddmadm.sha256.txt diff --git a/ddm/Cargo.toml b/ddm/Cargo.toml index 7856f5a78..373f90b06 100644 --- a/ddm/Cargo.toml +++ b/ddm/Cargo.toml @@ -21,10 +21,6 @@ hyper.workspace = true hyper-util.workspace = true http-body-util.workspace = true serde_json.workspace = true -libnet.workspace = true -dpd-client.workspace = true -opte-ioctl.workspace = true -oxide-vpc.workspace = true sled.workspace = true mg-common.workspace = true chrono.workspace = true @@ -36,3 +32,15 @@ uuid.workspace = true ddm-api.workspace = true ddm-types.workspace = true ddm-types-versions.workspace = true + +# illumos-only deps used by the routing state machine and platform sys layer. +# Gated by the `illumos` feature so non-illumos builds (e.g. Linux test +# fixtures running ddmd with `--no-state-machine`) link cleanly. +libnet = { workspace = true, optional = true } +dpd-client = { workspace = true, optional = true } +opte-ioctl = { workspace = true, optional = true } +oxide-vpc = { workspace = true, optional = true } + +[features] +default = ["illumos"] +illumos = ["dep:libnet", "dep:dpd-client", "dep:opte-ioctl", "dep:oxide-vpc"] diff --git a/ddm/src/admin.rs b/ddm/src/admin.rs index 26fe15e34..9ba23ebf3 100644 --- a/ddm/src/admin.rs +++ b/ddm/src/admin.rs @@ -12,8 +12,6 @@ use ddm_types::exchange::PathVector; use dropshot::ApiDescription; use dropshot::ApiDescriptionBuildErrors; use dropshot::ConfigDropshot; -use dropshot::ConfigLogging; -use dropshot::ConfigLoggingLevel; use dropshot::HttpError; use dropshot::HttpResponseOk; use dropshot::HttpResponseUpdatedNoContent; @@ -23,7 +21,7 @@ use dropshot::TypedBody; use mg_common::lock; use mg_common::net::{MulticastOrigin, TunnelOrigin}; use oxnet::Ipv6Net; -use slog::{Logger, error, info}; +use slog::{Logger, error, info, o}; use std::collections::{HashMap, HashSet}; use std::net::{IpAddr, SocketAddr, SocketAddrV4, SocketAddrV6}; use std::sync::Arc; @@ -35,6 +33,8 @@ use tokio::task::JoinHandle; pub const DDM_STATS_PORT: u16 = 8001; +const UNIT_API_SERVER: &str = "api_server"; + #[derive(Default)] pub struct RouterStats { pub originated_underlay_prefixes: AtomicU64, @@ -68,11 +68,11 @@ pub fn handler( ..Default::default() }; - let ds_log = ConfigLogging::StderrTerminal { - level: ConfigLoggingLevel::Error, - } - .to_logger("admin") - .map_err(|e| e.to_string())?; + let ds_log = log.new(o!( + "component" => crate::COMPONENT_DDM, + "module" => crate::MOD_ADMIN, + "unit" => UNIT_API_SERVER, + )); let api = api_description().map_err(|e| e.to_string())?; diff --git a/ddm/src/discovery/mod.rs b/ddm/src/discovery/mod.rs new file mode 100644 index 000000000..c3feb1bb9 --- /dev/null +++ b/ddm/src/discovery/mod.rs @@ -0,0 +1,118 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! This module implements the ddm router discovery mechanisms. These +//! mechanisms are responsible for three primary things +//! +//! 1. Soliciting other routers through UDP/IPv6 link local multicast. +//! 2. Sending out router advertisements in response to solicitations. +//! 3. Continuously soliciting link-local at a configurable rate to keep +//! sessions alive and sending out notifications when peering arrangements +//! expire due to not getting a solicitation response within a configurable +//! time threshold. +//! +//! [`Version`] and [`DiscoveryError`] are platform-agnostic and stay in this +//! module so the state machine type definitions in [`crate::sm`] continue to +//! compile when the routing runtime is gated out (e.g. Linux test fixtures +//! running ddmd with `--no-state-machine`). The runtime helpers that drive +//! the protocol over UDPv6 sockets live in the [`runtime`] submodule and +//! are illumos-only. +//! +//! ## Protocol +//! +//! The general sequence of events is depicted in the following diagram. +//! +//! *==========* *==========* +//! | violin | | piano | +//! *==========* *==========* +//! | | +//! | solicit(ff02::dd) | +//! |-------------------------->| +//! | advertise(fe80::47) | +//! |<--------------------------| +//! | | +//! | ... | +//! | | +//! | | +//! | solicit(ff02::dd) | +//! |-------------------------->| +//! | advertise(fe80::47) | +//! |<--------------------------| +//! | | +//! | solicit(ff02::dd) | +//! |-------------------------->| +//! | solicit(ff02::dd) | +//! |-------------------------->| +//! | solicit(ff02::dd) | +//! |-------------------------->| +//! | | +//! +----| | +//! expire | | | +//! piano | | | +//! +--->| | +//! +//! This shows violin sending a link-local multicast solicitation over the wire. +//! That solicitation is received by piano and piano respons with an +//! advertisement to violin's link-local unicast address. From this point +//! forward solicitations and responses continue. Each time violin gets a +//! response from piano, it updates the last seen timestamp for piano. If at +//! some point piano stops responding to solicitations and the last seen +//! timestamp is older than the expiration threshold, violin will expire the +//! session and send out a notification to the ddm state machine that started +//! it. Violin will continue to send out solicitations in case piano comes back. +//! +//! In the event that piano undergoes renumbering e.g. it's link-local unicast +//! address changes, this will be detected by violin and an advertisement update +//! will be sent to the ddm state machine through the notification channel +//! provided to the discovery subsystem. +//! +//! The DDM discovery multicast address is ff02::dd. Discovery packets are sent +//! over UDP using port number 0xddd. +//! +//! ## Packets +//! +//! Discovery packets follow a very simple format +//! +//! 1 2 3 +//! 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +//! +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +//! | version |S A r r r r r r| router kind | hostname len | +//! +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +//! | hostname : +//! +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +//! : .... : +//! +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +//! +//! The first byte indicates the version. The only valid version at present is +//! version 1. The second byte is a flags bitfield. The first position `S` +//! indicates a solicitation. The second position `A` indicates and +//! advertisement. All other positions are reserved for future use. The third +//! byte indicates the kind of router. Current values are 0 for a server router +//! and 1 for a transit routers. The fourth byte is a hostname length followed +//! directly by a hostname of up to 255 bytes in length. + +use thiserror::Error; + +#[cfg(all(feature = "illumos", target_os = "illumos"))] +mod runtime; + +#[cfg(all(feature = "illumos", target_os = "illumos"))] +pub(crate) use runtime::handler; + +#[derive(Debug, Copy, Clone)] +#[repr(u8)] +pub enum Version { + V2 = 2, + V3 = 3, + V4 = 4, +} + +#[derive(Error, Debug)] +pub enum DiscoveryError { + #[error("io error: {0}")] + Io(#[from] std::io::Error), + + #[error("serialization error: {0}")] + Serialization(#[from] ispf::Error), +} diff --git a/ddm/src/discovery.rs b/ddm/src/discovery/runtime.rs similarity index 73% rename from ddm/src/discovery.rs rename to ddm/src/discovery/runtime.rs index 2c94a08f2..8c6756646 100644 --- a/ddm/src/discovery.rs +++ b/ddm/src/discovery/runtime.rs @@ -2,92 +2,14 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -//! This file implements the ddm router discovery mechanisms. These mechanisms -//! are responsible for three primary things -//! -//! 1. Soliciting other routers through UDP/IPv6 link local multicast. -//! 2. Sending out router advertisements in response to solicitations. -//! 3. Continuously soliciting link-local at a configurable rate to keep -//! sessions alive and sending out notifications when peering arrangements -//! expire due to not getting a solicitation response within a configurable -//! time threshold. -//! -//! ## Protocol -//! -//! The general sequence of events is depicted in the following diagram. -//! -//! *==========* *==========* -//! | violin | | piano | -//! *==========* *==========* -//! | | -//! | solicit(ff02::dd) | -//! |-------------------------->| -//! | advertise(fe80::47) | -//! |<--------------------------| -//! | | -//! | ... | -//! | | -//! | | -//! | solicit(ff02::dd) | -//! |-------------------------->| -//! | advertise(fe80::47) | -//! |<--------------------------| -//! | | -//! | solicit(ff02::dd) | -//! |-------------------------->| -//! | solicit(ff02::dd) | -//! |-------------------------->| -//! | solicit(ff02::dd) | -//! |-------------------------->| -//! | | -//! +----| | -//! expire | | | -//! piano | | | -//! +--->| | -//! -//! This shows violin sending a link-local multicast solicitation over the wire. -//! That solicitation is received by piano and piano respons with an -//! advertisement to violin's link-local unicast address. From this point -//! forward solicitations and responses continue. Each time violin gets a -//! response from piano, it updates the last seen timestamp for piano. If at -//! some point piano stops responding to solicitations and the last seen -//! timestamp is older than the expiration threshold, violin will expire the -//! session and send out a notification to the ddm state machine that started -//! it. Violin will continue to send out solicitations in case piano comes back. -//! -//! In the event that piano undergoes renumbering e.g. it's link-local unicast -//! address changes, this will be detected by violin and an advertisement update -//! will be sent to the ddm state machine through the notification channel -//! provided to the discovery subsystem. -//! -//! The DDM discovery multicast address is ff02::dd. Discovery packets are sent -//! over UDP using port number 0xddd. -//! -//! ## Packets -//! -//! Discovery packets follow a very simple format -//! -//! 1 2 3 -//! 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -//! +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -//! | version |S A r r r r r r| router kind | hostname len | -//! +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -//! | hostname : -//! +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -//! : .... : -//! +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -//! -//! The first byte indicates the version. The only valid version at present is -//! version 1. The second byte is a flags bitfield. The first position `S` -//! indicates a solicitation. The second position `A` indicates and -//! advertisement. All other positions are reserved for future use. The third -//! byte indicates the kind of router. Current values are 0 for a server router -//! and 1 for a transit routers. The fourth byte is a hostname length followed -//! directly by a hostname of up to 255 bytes in length. +//! Runtime helpers for ddm router discovery: link-local UDPv6 sockets, +//! solicitation/advertisement loops, neighbor liveness, and the +//! [`handler`] entry point invoked by the routing state machine. +//! illumos-only. +use super::{DiscoveryError, Version}; use crate::db::Db; use crate::sm::{Config, Event, NeighborEvent, SessionStats}; -use crate::util::u8_slice_assume_init_ref; use crate::{dbg, err, inf, trc, wrn}; use ddm_types::db::{PeerInfo, PeerStatus, RouterKind}; use mg_common::lock; @@ -101,32 +23,22 @@ use std::sync::mpsc::Sender; use std::sync::{Arc, RwLock}; use std::thread::{sleep, spawn}; use std::time::{Duration, Instant}; -use thiserror::Error; const DDM_MADDR: Ipv6Addr = Ipv6Addr::new(0xff02, 0, 0, 0, 0, 0, 0, 0xdd); const DDM_PORT: u16 = 0xddd; const SOLICIT: u8 = 1; const ADVERTISE: u8 = 1 << 1; -#[derive(Debug, Copy, Clone)] -#[repr(u8)] -pub enum Version { - V2 = 2, - V3 = 3, - V4 = 4, -} - -#[derive(Error, Debug)] -pub enum DiscoveryError { - #[error("io error: {0}")] - Io(#[from] std::io::Error), - - #[error("serialization error: {0}")] - Serialization(#[from] ispf::Error), +/// Reinterpret an initialized prefix of `[MaybeUninit]` as `[u8]`. +/// +/// TODO: trade for `MaybeUninit::slice_assume_init_ref` when it stabilizes. +#[inline(always)] +const unsafe fn u8_slice_assume_init_ref(slice: &[MaybeUninit]) -> &[u8] { + unsafe { &*(slice as *const [MaybeUninit] as *const [u8]) } } #[derive(Debug, Serialize, Deserialize)] -pub struct DiscoveryPacket { +struct DiscoveryPacket { version: u8, flags: u8, kind: RouterKind, @@ -135,7 +47,7 @@ pub struct DiscoveryPacket { } impl DiscoveryPacket { - pub fn new_solicitation(hostname: String, kind: RouterKind) -> Self { + fn new_solicitation(hostname: String, kind: RouterKind) -> Self { Self { version: Version::V4 as u8, flags: SOLICIT, @@ -143,7 +55,7 @@ impl DiscoveryPacket { kind, } } - pub fn new_advertisement(hostname: String, kind: RouterKind) -> Self { + fn new_advertisement(hostname: String, kind: RouterKind) -> Self { Self { version: Version::V4 as u8, flags: ADVERTISE, @@ -151,18 +63,12 @@ impl DiscoveryPacket { kind, } } - pub fn is_solicitation(&self) -> bool { + fn is_solicitation(&self) -> bool { (self.flags & SOLICIT) != 0 } - pub fn is_advertisement(&self) -> bool { + fn is_advertisement(&self) -> bool { (self.flags & ADVERTISE) != 0 } - pub fn set_solicitation(&mut self) { - self.flags &= SOLICIT; - } - pub fn set_advertisement(&mut self) { - self.flags &= ADVERTISE; - } } #[derive(Clone)] diff --git a/ddm/src/exchange/mod.rs b/ddm/src/exchange/mod.rs new file mode 100644 index 000000000..0e384607a --- /dev/null +++ b/ddm/src/exchange/mod.rs @@ -0,0 +1,585 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! This module implements the ddm router prefix exchange mechanisms. These +//! mechanisms are responsible for announcing and withdrawing prefix sets to +//! and from peers. +//! +//! The module has a set of request initiators and request handlers for +//! announcing, withdrawing, and synchronizing routes with a given peer. +//! Communication between peers is over HTTP(s) requests. +//! +//! This module only contains basic mechanisms for prefix information exchange +//! with peers. How those mechanisms are used in the overall state machine +//! model of a ddm router is defined in the state machine implementation in +//! [`crate::sm`]. +//! +//! The wire types ([`Update`], [`UnderlayUpdate`], [`TunnelUpdate`], +//! [`MulticastUpdate`], and their versioned counterparts) are +//! platform-agnostic and stay in this module. The runtime helpers that drive +//! the HTTP exchange protocol and program forwarding state live in the +//! [`runtime`] submodule and are illumos-only, since they call into +//! [`crate::sys`] to install routes. + +use ddm_types::exchange::{ + MulticastPathHop, MulticastPathVector, PathVector, PathVectorV2, +}; +use mg_common::net::{TunnelOrigin, TunnelOriginV2}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::collections::HashSet; +use thiserror::Error; + +#[cfg(all(feature = "illumos", target_os = "illumos"))] +mod runtime; + +#[cfg(all(feature = "illumos", target_os = "illumos"))] +pub(crate) use runtime::{ + announce_multicast, announce_tunnel, announce_underlay, do_pull_v4, + handler, pull, withdraw_multicast, withdraw_tunnel, withdraw_underlay, +}; + +/// THIS TYPE IS FOR DDM PROTOCOL VERSION 1. IT SHALL NEVER CHANGE. THIS TYPE +/// CAN BE REMOVED WHEN DDMV1 CLIENTS AND SERVERS NO LONGER EXIST BUT ITS +/// DEFINITION SHALL NEVER CHANGE. +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] +pub struct UpdateV1 { + pub announce: HashSet, + pub withdraw: HashSet, +} + +/// THIS TYPE IS FOR DDM PROTOCOL VERSION 2. IT SHALL NEVER CHANGE. THIS TYPE +/// CAN BE REMOVED WHEN DDMV2 CLIENTS AND SERVERS NO LONGER EXIST BUT ITS +/// DEFINITION SHALL NEVER CHANGE. +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] +pub struct UpdateV2 { + pub underlay: Option, + pub tunnel: Option, +} + +/// THIS TYPE IS FOR DDM PROTOCOL VERSION 3. IT SHALL NEVER CHANGE. THIS TYPE +/// CAN BE REMOVED WHEN DDMV3 CLIENTS AND SERVERS NO LONGER EXIST BUT ITS +/// DEFINITION SHALL NEVER CHANGE. +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] +pub struct UpdateV3 { + pub underlay: Option, + pub tunnel: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] +pub struct Update { + pub underlay: Option, + pub tunnel: Option, + pub multicast: Option, +} + +impl From for Update { + fn from(value: UpdateV1) -> Self { + Update { + tunnel: None, + underlay: Some(UnderlayUpdate { + announce: value.announce, + withdraw: value.withdraw, + }), + multicast: None, + } + } +} + +impl From for Update { + fn from(value: UpdateV2) -> Self { + Update { + tunnel: value.tunnel.map(TunnelUpdate::from), + underlay: value.underlay.map(UnderlayUpdate::from), + // V2 protocol doesn't support multicast + multicast: None, + } + } +} + +impl From for UpdateV1 { + fn from(value: Update) -> Self { + let (announce, withdraw) = match value.underlay { + Some(underlay) => (underlay.announce, underlay.withdraw), + None => (HashSet::new(), HashSet::new()), + }; + UpdateV1 { announce, withdraw } + } +} + +impl From for UpdateV2 { + fn from(value: Update) -> Self { + UpdateV2 { + tunnel: value.tunnel.map(TunnelUpdateV2::from), + underlay: value.underlay.map(UnderlayUpdateV2::from), + } + } +} + +impl From for Update { + fn from(value: UpdateV3) -> Self { + Update { + underlay: value.underlay, + tunnel: value.tunnel, + multicast: None, + } + } +} + +impl From for UpdateV3 { + fn from(value: Update) -> Self { + UpdateV3 { + underlay: value.underlay, + tunnel: value.tunnel, + } + } +} + +impl From for Update { + fn from(u: UnderlayUpdate) -> Self { + Update { + underlay: Some(u), + tunnel: None, + multicast: None, + } + } +} + +impl From for Update { + fn from(t: TunnelUpdate) -> Self { + Update { + underlay: None, + tunnel: Some(t), + multicast: None, + } + } +} + +impl From for Update { + fn from(m: MulticastUpdate) -> Self { + Update { + underlay: None, + tunnel: None, + multicast: Some(m), + } + } +} + +/// THIS TYPE IS FOR DDM PROTOCOL VERSION 3. IT SHALL NEVER CHANGE. THIS TYPE +/// CAN BE REMOVED WHEN DDMV3 CLIENTS AND SERVERS NO LONGER EXIST BUT ITS +/// DEFINITION SHALL NEVER CHANGE. +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] +pub struct PullResponseV3 { + pub underlay: Option>, + pub tunnel: Option>, +} + +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] +pub struct PullResponse { + pub underlay: Option>, + pub tunnel: Option>, + pub multicast: Option>, +} + +/// THIS TYPE IS FOR DDM PROTOCOL VERSION 2. IT SHALL NEVER CHANGE. THIS TYPE +/// CAN BE REMOVED WHEN DDMV2 CLIENTS AND SERVERS NO LONGER EXIST BUT ITS +/// DEFINITION SHALL NEVER CHANGE. +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] +pub struct PullResponseV2 { + pub underlay: Option>, + pub tunnel: Option>, +} + +impl From for PullResponse { + fn from(value: PullResponseV2) -> Self { + PullResponse { + underlay: value + .underlay + .map(|x| x.into_iter().map(PathVector::from).collect()), + tunnel: value + .tunnel + .map(|x| x.into_iter().map(TunnelOrigin::from).collect()), + // V2 protocol doesn't support multicast + multicast: None, + } + } +} + +impl From for PullResponse { + fn from(value: PullResponseV3) -> Self { + PullResponse { + underlay: value.underlay, + tunnel: value.tunnel, + multicast: None, + } + } +} + +impl From> for PullResponse { + fn from(value: HashSet) -> Self { + PullResponse { + underlay: Some(value), + tunnel: None, + multicast: None, + } + } +} + +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] +pub struct UnderlayUpdate { + pub announce: HashSet, + pub withdraw: HashSet, +} + +/// THIS TYPE IS FOR DDM PROTOCOL VERSION 2. IT SHALL NEVER CHANGE. THIS TYPE +/// CAN BE REMOVED WHEN DDMV2 CLIENTS AND SERVERS NO LONGER EXIST BUT ITS +/// DEFINITION SHALL NEVER CHANGE. +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] +pub struct UnderlayUpdateV2 { + pub announce: HashSet, + pub withdraw: HashSet, +} + +impl From for UnderlayUpdateV2 { + fn from(value: UnderlayUpdate) -> Self { + UnderlayUpdateV2 { + announce: value + .announce + .into_iter() + .map(PathVectorV2::from) + .collect(), + withdraw: value + .withdraw + .into_iter() + .map(PathVectorV2::from) + .collect(), + } + } +} + +impl From for UnderlayUpdate { + fn from(value: UnderlayUpdateV2) -> Self { + UnderlayUpdate { + announce: value + .announce + .into_iter() + .map(PathVector::from) + .collect(), + withdraw: value + .withdraw + .into_iter() + .map(PathVector::from) + .collect(), + } + } +} + +impl UnderlayUpdate { + pub fn announce(prefixes: HashSet) -> Self { + Self { + announce: prefixes, + ..Default::default() + } + } + pub fn withdraw(prefixes: HashSet) -> Self { + Self { + withdraw: prefixes, + ..Default::default() + } + } + pub fn with_path_element(&self, element: String) -> Self { + Self { + announce: self + .announce + .iter() + .map(|x| { + let mut pv = x.clone(); + pv.path.push(element.clone()); + pv + }) + .collect(), + withdraw: self + .withdraw + .iter() + .map(|x| { + let mut pv = x.clone(); + pv.path.push(element.clone()); + pv + }) + .collect(), + } + } +} + +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] +pub struct TunnelUpdate { + pub announce: HashSet, + pub withdraw: HashSet, +} + +/// THIS TYPE IS FOR DDM PROTOCOL VERSION 2. IT SHALL NEVER CHANGE. THIS TYPE +/// CAN BE REMOVED WHEN DDMV2 CLIENTS AND SERVERS NO LONGER EXIST BUT ITS +/// DEFINITION SHALL NEVER CHANGE. +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] +pub struct TunnelUpdateV2 { + pub announce: HashSet, + pub withdraw: HashSet, +} + +impl From for TunnelUpdate { + fn from(value: TunnelUpdateV2) -> Self { + TunnelUpdate { + announce: value + .announce + .into_iter() + .map(TunnelOrigin::from) + .collect(), + withdraw: value + .withdraw + .into_iter() + .map(TunnelOrigin::from) + .collect(), + } + } +} + +impl From for TunnelUpdateV2 { + fn from(value: TunnelUpdate) -> Self { + TunnelUpdateV2 { + announce: value + .announce + .into_iter() + .map(TunnelOriginV2::from) + .collect(), + withdraw: value + .withdraw + .into_iter() + .map(TunnelOriginV2::from) + .collect(), + } + } +} + +impl TunnelUpdate { + pub fn announce(prefixes: HashSet) -> Self { + Self { + announce: prefixes, + ..Default::default() + } + } + pub fn withdraw(prefixes: HashSet) -> Self { + Self { + withdraw: prefixes, + ..Default::default() + } + } +} + +/// Multicast group subscription updates. +/// +/// Each entry carries a [`MulticastPathVector`] containing a +/// [`MulticastOrigin`] (overlay group + ff04::/64 underlay mapping) +/// and the path vector for loop detection. +/// +/// [`MulticastOrigin`]: mg_common::net::MulticastOrigin +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] +pub struct MulticastUpdate { + pub announce: HashSet, + pub withdraw: HashSet, +} + +impl MulticastUpdate { + pub fn announce(groups: HashSet) -> Self { + Self { + announce: groups, + ..Default::default() + } + } + pub fn withdraw(groups: HashSet) -> Self { + Self { + withdraw: groups, + ..Default::default() + } + } + + /// Add a hop to all path vectors in this update. + pub fn with_hop(&self, hop: MulticastPathHop) -> Self { + Self { + announce: self + .announce + .iter() + .map(|pv| pv.with_hop(hop.clone())) + .collect(), + withdraw: self + .withdraw + .iter() + .map(|pv| pv.with_hop(hop.clone())) + .collect(), + } + } +} + +#[derive(Error, Debug)] +pub enum ExchangeError { + #[error("io error: {0}")] + Io(#[from] std::io::Error), + + #[error("hyper error: {0}")] + Hyper(#[from] hyper::Error), + + #[error("hyper client error: {0}")] + HyperClient(#[from] hyper_util::client::legacy::Error), + + #[error("timeout error: {0}")] + Timeout(#[from] tokio::time::error::Elapsed), + + #[error("json error: {0}")] + SerdeJson(#[from] serde_json::Error), +} + +#[cfg(test)] +mod tests { + use super::*; + use ddm_types::exchange::MulticastPathHop; + use mg_common::net::{MulticastOrigin, UnderlayMulticastIpv6, Vni}; + use std::net::Ipv6Addr; + + fn sample_multicast_update() -> MulticastUpdate { + let origin = MulticastOrigin { + overlay_group: "233.252.0.1".parse().unwrap(), + underlay_group: UnderlayMulticastIpv6::new( + "ff04::1".parse().unwrap(), + ) + .unwrap(), + vni: Vni::try_from(77u32).unwrap(), + metric: 0, + source: None, + }; + let pv = MulticastPathVector { + origin, + path: vec![MulticastPathHop::new( + "router-1".into(), + Ipv6Addr::LOCALHOST, + )], + }; + MulticastUpdate::announce([pv].into_iter().collect()) + } + + #[test] + fn v4_update_round_trips() { + let update = Update { + underlay: None, + tunnel: None, + multicast: Some(sample_multicast_update()), + }; + let json = serde_json::to_string(&update).unwrap(); + let back: Update = serde_json::from_str(&json).unwrap(); + assert!(back.multicast.is_some()); + assert_eq!(back.multicast.unwrap().announce.len(), 1,); + } + + #[test] + fn v4_update_deserializes_as_v3_drops_multicast() { + let update = Update { + underlay: None, + tunnel: None, + multicast: Some(sample_multicast_update()), + }; + let json = serde_json::to_string(&update).unwrap(); + // A V3 peer would deserialize this as UpdateV3, silently + // dropping the unknown multicast field. + let v3: UpdateV3 = serde_json::from_str(&json).unwrap(); + assert!(v3.underlay.is_none()); + assert!(v3.tunnel.is_none()); + } + + #[test] + fn v3_update_deserializes_as_v4_multicast_none() { + let v3 = UpdateV3 { + underlay: None, + tunnel: None, + }; + let json = serde_json::to_string(&v3).unwrap(); + // A V4 peer receiving a V3 update gets multicast: None. + let update: Update = serde_json::from_str(&json).unwrap(); + assert!(update.multicast.is_none()); + } + + #[test] + fn v4_pull_response_round_trips() { + let origin = MulticastOrigin { + overlay_group: "ff0e::1".parse().unwrap(), + underlay_group: UnderlayMulticastIpv6::new( + "ff04::2".parse().unwrap(), + ) + .unwrap(), + vni: Vni::try_from(77u32).unwrap(), + metric: 0, + source: None, + }; + let pv = MulticastPathVector { + origin, + path: vec![], + }; + let resp = PullResponse { + underlay: None, + tunnel: None, + multicast: Some([pv].into_iter().collect()), + }; + let json = serde_json::to_string(&resp).unwrap(); + let back: PullResponse = serde_json::from_str(&json).unwrap(); + assert!(back.multicast.is_some()); + } + + #[test] + fn v4_pull_response_deserializes_as_v3() { + let origin = MulticastOrigin { + overlay_group: "233.252.0.1".parse().unwrap(), + underlay_group: UnderlayMulticastIpv6::new( + "ff04::1".parse().unwrap(), + ) + .unwrap(), + vni: Vni::try_from(77u32).unwrap(), + metric: 0, + source: None, + }; + let pv = MulticastPathVector { + origin, + path: vec![], + }; + let resp = PullResponse { + underlay: None, + tunnel: None, + multicast: Some([pv].into_iter().collect()), + }; + let json = serde_json::to_string(&resp).unwrap(); + // V3 peer drops the multicast field. + let v3: PullResponseV3 = serde_json::from_str(&json).unwrap(); + assert!(v3.underlay.is_none()); + assert!(v3.tunnel.is_none()); + } + + #[test] + fn v3_pull_response_deserializes_as_v4() { + let v3 = PullResponseV3 { + underlay: None, + tunnel: None, + }; + let json = serde_json::to_string(&v3).unwrap(); + let resp: PullResponse = serde_json::from_str(&json).unwrap(); + assert!(resp.multicast.is_none()); + } + + #[test] + fn from_conversions_strip_multicast() { + let update = Update { + underlay: None, + tunnel: None, + multicast: Some(sample_multicast_update()), + }; + let v3 = UpdateV3::from(update); + let back = Update::from(v3); + assert!(back.multicast.is_none()); + } +} diff --git a/ddm/src/exchange.rs b/ddm/src/exchange/runtime.rs similarity index 58% rename from ddm/src/exchange.rs rename to ddm/src/exchange/runtime.rs index 9e696e371..ea069f1d8 100644 --- a/ddm/src/exchange.rs +++ b/ddm/src/exchange/runtime.rs @@ -2,19 +2,15 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -//! This file implements the ddm router prefix exchange mechanisms. These -//! mechanisms are responsible for announcing and withdrawing prefix sets to and -//! from peers. -//! -//! This file has a set of request initiators and request handlers for -//! announcing, withdrawing and synchronizing routes with a a given peer. -//! Communication between peers is over HTTP(s) requests. -//! -//! This file only contains basic mechanisms for prefix information exchange -//! with peers. How those mechanisms are used in the overall state machine model -//! of a ddm router is defined in the state machine implementation in sm.rs. -//! - +//! Runtime helpers for the ddm prefix exchange protocol: HTTP push/pull +//! initiators and dropshot endpoint handlers, and the route programming +//! plumbing that drains received updates into the local DB and the +//! forwarding platform via [`crate::sys`]. illumos-only. + +use super::{ + ExchangeError, MulticastUpdate, PullResponse, PullResponseV2, + PullResponseV3, TunnelUpdate, UnderlayUpdate, Update, UpdateV2, UpdateV3, +}; use crate::db::{Route, effective_route_set}; use crate::discovery::Version; use crate::sm::{Config, Event, PeerEvent, SmContext}; @@ -25,8 +21,6 @@ use ddm_types::exchange::{ }; use dropshot::ApiDescription; use dropshot::ConfigDropshot; -use dropshot::ConfigLogging; -use dropshot::ConfigLoggingLevel; use dropshot::HttpError; use dropshot::HttpResponseOk; use dropshot::HttpResponseUpdatedNoContent; @@ -39,18 +33,17 @@ use hyper::body::Bytes; use hyper_util::client::legacy::Client; use hyper_util::rt::TokioExecutor; use mg_common::net::{TunnelOrigin, TunnelOriginV2}; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; -use slog::Logger; +use slog::{Logger, o}; use std::collections::HashSet; use std::net::{Ipv6Addr, SocketAddrV6}; use std::sync::Arc; use std::sync::atomic::Ordering; use std::time::Duration; -use thiserror::Error; use tokio::sync::Mutex; use tokio::time::timeout; +const UNIT_EXCHANGE_SERVER: &str = "exchange_server"; + #[derive(Clone)] pub struct HandlerContext { ctx: SmContext, @@ -58,123 +51,10 @@ pub struct HandlerContext { log: Logger, } -/// THIS TYPE IS FOR DDM PROTOCOL VERSION 1. IT SHALL NEVER CHANGE. THIS TYPE -/// CAN BE REMOVED WHEN DDMV1 CLIENTS AND SERVERS NO LONGER EXIST BUT ITS -/// DEFINITION SHALL NEVER CHANGE. -#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] -pub struct UpdateV1 { - pub announce: HashSet, - pub withdraw: HashSet, -} - -/// THIS TYPE IS FOR DDM PROTOCOL VERSION 2. IT SHALL NEVER CHANGE. THIS TYPE -/// CAN BE REMOVED WHEN DDMV2 CLIENTS AND SERVERS NO LONGER EXIST BUT ITS -/// DEFINITION SHALL NEVER CHANGE. -#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] -pub struct UpdateV2 { - pub underlay: Option, - pub tunnel: Option, -} - -/// THIS TYPE IS FOR DDM PROTOCOL VERSION 3. IT SHALL NEVER CHANGE. THIS TYPE -/// CAN BE REMOVED WHEN DDMV3 CLIENTS AND SERVERS NO LONGER EXIST BUT ITS -/// DEFINITION SHALL NEVER CHANGE. -#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] -pub struct UpdateV3 { - pub underlay: Option, - pub tunnel: Option, -} - -#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] -pub struct Update { - pub underlay: Option, - pub tunnel: Option, - pub multicast: Option, -} - -impl From for Update { - fn from(value: UpdateV1) -> Self { - Update { - tunnel: None, - underlay: Some(UnderlayUpdate { - announce: value.announce, - withdraw: value.withdraw, - }), - multicast: None, - } - } -} - -impl From for Update { - fn from(value: UpdateV2) -> Self { - Update { - tunnel: value.tunnel.map(TunnelUpdate::from), - underlay: value.underlay.map(UnderlayUpdate::from), - // V2 protocol doesn't support multicast - multicast: None, - } - } -} - -impl From for UpdateV1 { - fn from(value: Update) -> Self { - let (announce, withdraw) = match value.underlay { - Some(underlay) => (underlay.announce, underlay.withdraw), - None => (HashSet::new(), HashSet::new()), - }; - UpdateV1 { announce, withdraw } - } -} - -impl From for UpdateV2 { - fn from(value: Update) -> Self { - UpdateV2 { - tunnel: value.tunnel.map(TunnelUpdateV2::from), - underlay: value.underlay.map(UnderlayUpdateV2::from), - } - } -} - -impl From for Update { - fn from(value: UpdateV3) -> Self { - Update { - underlay: value.underlay, - tunnel: value.tunnel, - multicast: None, - } - } -} - -impl From for UpdateV3 { - fn from(value: Update) -> Self { - UpdateV3 { - underlay: value.underlay, - tunnel: value.tunnel, - } - } -} - -impl From for Update { - fn from(u: UnderlayUpdate) -> Self { - Update { - underlay: Some(u), - tunnel: None, - multicast: None, - } - } -} - -impl From for Update { - fn from(t: TunnelUpdate) -> Self { - Update { - underlay: None, - tunnel: Some(t), - multicast: None, - } - } -} - impl Update { + /// Build an `Update` whose underlay/tunnel/multicast halves carry the + /// announcements from `pr`. Used by [`pull`] to project a pull response + /// back into the update event stream. fn announce(pr: PullResponse) -> Self { Self { underlay: pr.underlay.map(UnderlayUpdate::announce), @@ -184,288 +64,6 @@ impl Update { } } -impl From for Update { - fn from(m: MulticastUpdate) -> Self { - Update { - underlay: None, - tunnel: None, - multicast: Some(m), - } - } -} - -/// THIS TYPE IS FOR DDM PROTOCOL VERSION 3. IT SHALL NEVER CHANGE. THIS TYPE -/// CAN BE REMOVED WHEN DDMV3 CLIENTS AND SERVERS NO LONGER EXIST BUT ITS -/// DEFINITION SHALL NEVER CHANGE. -#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] -pub struct PullResponseV3 { - pub underlay: Option>, - pub tunnel: Option>, -} - -#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] -pub struct PullResponse { - pub underlay: Option>, - pub tunnel: Option>, - pub multicast: Option>, -} - -/// THIS TYPE IS FOR DDM PROTOCOL VERSION 2. IT SHALL NEVER CHANGE. THIS TYPE -/// CAN BE REMOVED WHEN DDMV2 CLIENTS AND SERVERS NO LONGER EXIST BUT ITS -/// DEFINITION SHALL NEVER CHANGE. -#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] -pub struct PullResponseV2 { - pub underlay: Option>, - pub tunnel: Option>, -} - -impl From for PullResponse { - fn from(value: PullResponseV2) -> Self { - PullResponse { - underlay: value - .underlay - .map(|x| x.into_iter().map(PathVector::from).collect()), - tunnel: value - .tunnel - .map(|x| x.into_iter().map(TunnelOrigin::from).collect()), - // V2 protocol doesn't support multicast - multicast: None, - } - } -} - -impl From for PullResponse { - fn from(value: PullResponseV3) -> Self { - PullResponse { - underlay: value.underlay, - tunnel: value.tunnel, - multicast: None, - } - } -} - -impl From> for PullResponse { - fn from(value: HashSet) -> Self { - PullResponse { - underlay: Some(value), - tunnel: None, - multicast: None, - } - } -} - -#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] -pub struct UnderlayUpdate { - pub announce: HashSet, - pub withdraw: HashSet, -} - -/// THIS TYPE IS FOR DDM PROTOCOL VERSION 2. IT SHALL NEVER CHANGE. THIS TYPE -/// CAN BE REMOVED WHEN DDMV2 CLIENTS AND SERVERS NO LONGER EXIST BUT ITS -/// DEFINITION SHALL NEVER CHANGE. -#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] -pub struct UnderlayUpdateV2 { - pub announce: HashSet, - pub withdraw: HashSet, -} - -impl From for UnderlayUpdateV2 { - fn from(value: UnderlayUpdate) -> Self { - UnderlayUpdateV2 { - announce: value - .announce - .into_iter() - .map(PathVectorV2::from) - .collect(), - withdraw: value - .withdraw - .into_iter() - .map(PathVectorV2::from) - .collect(), - } - } -} - -impl From for UnderlayUpdate { - fn from(value: UnderlayUpdateV2) -> Self { - UnderlayUpdate { - announce: value - .announce - .into_iter() - .map(PathVector::from) - .collect(), - withdraw: value - .withdraw - .into_iter() - .map(PathVector::from) - .collect(), - } - } -} - -impl UnderlayUpdate { - pub fn announce(prefixes: HashSet) -> Self { - Self { - announce: prefixes, - ..Default::default() - } - } - pub fn withdraw(prefixes: HashSet) -> Self { - Self { - withdraw: prefixes, - ..Default::default() - } - } - pub fn with_path_element(&self, element: String) -> Self { - Self { - announce: self - .announce - .iter() - .map(|x| { - let mut pv = x.clone(); - pv.path.push(element.clone()); - pv - }) - .collect(), - withdraw: self - .withdraw - .iter() - .map(|x| { - let mut pv = x.clone(); - pv.path.push(element.clone()); - pv - }) - .collect(), - } - } -} - -#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] -pub struct TunnelUpdate { - pub announce: HashSet, - pub withdraw: HashSet, -} - -/// THIS TYPE IS FOR DDM PROTOCOL VERSION 2. IT SHALL NEVER CHANGE. THIS TYPE -/// CAN BE REMOVED WHEN DDMV2 CLIENTS AND SERVERS NO LONGER EXIST BUT ITS -/// DEFINITION SHALL NEVER CHANGE. -#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] -pub struct TunnelUpdateV2 { - pub announce: HashSet, - pub withdraw: HashSet, -} - -impl From for TunnelUpdate { - fn from(value: TunnelUpdateV2) -> Self { - TunnelUpdate { - announce: value - .announce - .into_iter() - .map(TunnelOrigin::from) - .collect(), - withdraw: value - .withdraw - .into_iter() - .map(TunnelOrigin::from) - .collect(), - } - } -} - -impl From for TunnelUpdateV2 { - fn from(value: TunnelUpdate) -> Self { - TunnelUpdateV2 { - announce: value - .announce - .into_iter() - .map(TunnelOriginV2::from) - .collect(), - withdraw: value - .withdraw - .into_iter() - .map(TunnelOriginV2::from) - .collect(), - } - } -} - -impl TunnelUpdate { - pub fn announce(prefixes: HashSet) -> Self { - Self { - announce: prefixes, - ..Default::default() - } - } - pub fn withdraw(prefixes: HashSet) -> Self { - Self { - withdraw: prefixes, - ..Default::default() - } - } -} - -/// Multicast group subscription updates. -/// -/// Each entry carries a [`MulticastPathVector`] containing a -/// [`MulticastOrigin`] (overlay group + ff04::/64 underlay mapping) -/// and the path vector for loop detection. -/// -/// [`MulticastOrigin`]: mg_common::net::MulticastOrigin -#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] -pub struct MulticastUpdate { - pub announce: HashSet, - pub withdraw: HashSet, -} - -impl MulticastUpdate { - pub fn announce(groups: HashSet) -> Self { - Self { - announce: groups, - ..Default::default() - } - } - pub fn withdraw(groups: HashSet) -> Self { - Self { - withdraw: groups, - ..Default::default() - } - } - - /// Add a hop to all path vectors in this update. - pub fn with_hop(&self, hop: MulticastPathHop) -> Self { - Self { - announce: self - .announce - .iter() - .map(|pv| pv.with_hop(hop.clone())) - .collect(), - withdraw: self - .withdraw - .iter() - .map(|pv| pv.with_hop(hop.clone())) - .collect(), - } - } -} - -#[derive(Error, Debug)] -pub enum ExchangeError { - #[error("io error: {0}")] - Io(#[from] std::io::Error), - - #[error("hyper error: {0}")] - Hyper(#[from] hyper::Error), - - #[error("hyper client error: {0}")] - HyperClient(#[from] hyper_util::client::legacy::Error), - - #[error("timeout error: {0}")] - Timeout(#[from] tokio::time::error::Elapsed), - - #[error("json error: {0}")] - SerdeJson(#[from] serde_json::Error), -} - pub(crate) fn announce_underlay( ctx: &SmContext, config: Config, @@ -710,11 +308,11 @@ pub fn handler( ..Default::default() }; - let ds_log = ConfigLogging::StderrTerminal { - level: ConfigLoggingLevel::Error, - } - .to_logger("exchange") - .map_err(|e| e.to_string())?; + let ds_log = log.new(o!( + "component" => crate::COMPONENT_DDM, + "module" => crate::MOD_EXCHANGE, + "unit" => UNIT_EXCHANGE_SERVER, + )); inf!(log, ctx.config.if_name, "exchange: listening on {}", sa); @@ -1201,15 +799,6 @@ fn handle_underlay_update(update: &UnderlayUpdate, ctx: &HandlerContext) { .store(ctx.ctx.db.imported_count() as u64, Ordering::Relaxed); } -/// Handle multicast group subscription updates from a peer. -/// -/// Validation uses path-vector-based RPF rather than unicast-RIB RPF. -/// DDM operates on the underlay while multicast sources are overlay -/// addresses, so traditional (S,G) RPF against the unicast RIB does not -/// apply at this layer. The MRIB RPF module in rdb handles that check -/// before routes are originated into DDM. At the DDM exchange level, -/// the path vector provides loop detection and carries topology -/// information for replication optimization (RFD 488). fn handle_multicast_update(update: &MulticastUpdate, ctx: &HandlerContext) { let db = &ctx.ctx.db; let hostname = &ctx.ctx.hostname; @@ -1256,149 +845,3 @@ fn handle_multicast_update(update: &MulticastUpdate, ctx: &HandlerContext) { // Learned state is queryable via the DDM admin API (get_multicast_groups). db.update_imported_mcast(&import, &remove); } - -#[cfg(test)] -mod test { - use super::*; - use ddm_types::exchange::MulticastPathHop; - use mg_common::net::{MulticastOrigin, UnderlayMulticastIpv6, Vni}; - use std::net::Ipv6Addr; - - fn sample_multicast_update() -> MulticastUpdate { - let origin = MulticastOrigin { - overlay_group: "233.252.0.1".parse().unwrap(), - underlay_group: UnderlayMulticastIpv6::new( - "ff04::1".parse().unwrap(), - ) - .unwrap(), - vni: Vni::try_from(77u32).unwrap(), - metric: 0, - source: None, - }; - let pv = MulticastPathVector { - origin, - path: vec![MulticastPathHop::new( - "router-1".into(), - Ipv6Addr::LOCALHOST, - )], - }; - MulticastUpdate::announce([pv].into_iter().collect()) - } - - #[test] - fn v4_update_round_trips() { - let update = Update { - underlay: None, - tunnel: None, - multicast: Some(sample_multicast_update()), - }; - let json = serde_json::to_string(&update).unwrap(); - let back: Update = serde_json::from_str(&json).unwrap(); - assert!(back.multicast.is_some()); - assert_eq!(back.multicast.unwrap().announce.len(), 1,); - } - - #[test] - fn v4_update_deserializes_as_v3_drops_multicast() { - let update = Update { - underlay: None, - tunnel: None, - multicast: Some(sample_multicast_update()), - }; - let json = serde_json::to_string(&update).unwrap(); - // A V3 peer would deserialize this as UpdateV3, silently - // dropping the unknown multicast field. - let v3: UpdateV3 = serde_json::from_str(&json).unwrap(); - assert!(v3.underlay.is_none()); - assert!(v3.tunnel.is_none()); - } - - #[test] - fn v3_update_deserializes_as_v4_multicast_none() { - let v3 = UpdateV3 { - underlay: None, - tunnel: None, - }; - let json = serde_json::to_string(&v3).unwrap(); - // A V4 peer receiving a V3 update gets multicast: None. - let update: Update = serde_json::from_str(&json).unwrap(); - assert!(update.multicast.is_none()); - } - - #[test] - fn v4_pull_response_round_trips() { - let origin = MulticastOrigin { - overlay_group: "ff0e::1".parse().unwrap(), - underlay_group: UnderlayMulticastIpv6::new( - "ff04::2".parse().unwrap(), - ) - .unwrap(), - vni: Vni::try_from(77u32).unwrap(), - metric: 0, - source: None, - }; - let pv = MulticastPathVector { - origin, - path: vec![], - }; - let resp = PullResponse { - underlay: None, - tunnel: None, - multicast: Some([pv].into_iter().collect()), - }; - let json = serde_json::to_string(&resp).unwrap(); - let back: PullResponse = serde_json::from_str(&json).unwrap(); - assert!(back.multicast.is_some()); - } - - #[test] - fn v4_pull_response_deserializes_as_v3() { - let origin = MulticastOrigin { - overlay_group: "233.252.0.1".parse().unwrap(), - underlay_group: UnderlayMulticastIpv6::new( - "ff04::1".parse().unwrap(), - ) - .unwrap(), - vni: Vni::try_from(77u32).unwrap(), - metric: 0, - source: None, - }; - let pv = MulticastPathVector { - origin, - path: vec![], - }; - let resp = PullResponse { - underlay: None, - tunnel: None, - multicast: Some([pv].into_iter().collect()), - }; - let json = serde_json::to_string(&resp).unwrap(); - // V3 peer drops the multicast field. - let v3: PullResponseV3 = serde_json::from_str(&json).unwrap(); - assert!(v3.underlay.is_none()); - assert!(v3.tunnel.is_none()); - } - - #[test] - fn v3_pull_response_deserializes_as_v4() { - let v3 = PullResponseV3 { - underlay: None, - tunnel: None, - }; - let json = serde_json::to_string(&v3).unwrap(); - let resp: PullResponse = serde_json::from_str(&json).unwrap(); - assert!(resp.multicast.is_none()); - } - - #[test] - fn from_conversions_strip_multicast() { - let update = Update { - underlay: None, - tunnel: None, - multicast: Some(sample_multicast_update()), - }; - let v3 = UpdateV3::from(update); - let back = Update::from(v3); - assert!(back.multicast.is_none()); - } -} diff --git a/ddm/src/lib.rs b/ddm/src/lib.rs index 7699ce439..a382d5188 100644 --- a/ddm/src/lib.rs +++ b/ddm/src/lib.rs @@ -8,10 +8,15 @@ pub mod discovery; pub mod exchange; pub mod oxstats; pub mod sm; +#[cfg(all(feature = "illumos", target_os = "illumos"))] pub mod sys; -mod util; + +pub const COMPONENT_DDM: &str = "ddm"; +pub const MOD_ADMIN: &str = "admin"; +pub const MOD_EXCHANGE: &str = "exchange"; /// Returns `None` if the set is empty, otherwise `Some(s)`. +#[cfg(all(feature = "illumos", target_os = "illumos"))] pub(crate) fn non_empty( set: std::collections::HashSet, ) -> Option> { diff --git a/ddm/src/sm/mod.rs b/ddm/src/sm/mod.rs new file mode 100644 index 000000000..51a2c1bbe --- /dev/null +++ b/ddm/src/sm/mod.rs @@ -0,0 +1,195 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! State machine type definitions and the [`StateMachine`] handle. The +//! routing state machine implementation (discovery, solicit, exchange) lives +//! in the [`state`] submodule and is illumos-only, since it programs kernel +//! routes via [`crate::sys`] and reads interface addressing through `libnet`. + +use crate::db::Db; +use crate::discovery::{self, Version}; +use crate::exchange::Update; +use ddm_types::db::RouterKind; +use mg_common::net::{MulticastOrigin, TunnelOrigin}; +use oxnet::Ipv6Net; +use slog::Logger; +use std::collections::HashSet; +use std::net::Ipv6Addr; +use std::sync::atomic::AtomicU64; +use std::sync::mpsc::{Receiver, Sender}; +use std::sync::{Arc, Mutex}; +use thiserror::Error; + +#[cfg(all(feature = "illumos", target_os = "illumos"))] +mod state; + +#[derive(Debug)] +pub enum AdminEvent { + /// Announce a set of IPv6 prefixes + Announce(PrefixSet), + + /// Withdraw a set of IPv6 prefixes + Withdraw(PrefixSet), + + /// Expire the peer at the specified address + Expire(Ipv6Addr), + + /// Synchronize with active peers by pulling their prefixes. + Sync, +} + +#[derive(Debug)] +pub enum PrefixSet { + Underlay(HashSet), + Tunnel(HashSet), + Multicast(HashSet), +} + +#[derive(Debug)] +pub enum PeerEvent { + Push(Arc), +} + +#[derive(Debug)] +pub enum NeighborEvent { + Advertise((Ipv6Addr, Version)), + SolicitFail, + Expire, +} + +#[derive(Debug)] +pub enum Event { + Neighbor(NeighborEvent), + Peer(PeerEvent), + Admin(AdminEvent), +} + +impl From for Event { + fn from(e: NeighborEvent) -> Self { + Self::Neighbor(e) + } +} + +impl From for Event { + fn from(e: PeerEvent) -> Self { + Self::Peer(e) + } +} + +impl From for Event { + fn from(e: AdminEvent) -> Self { + Self::Admin(e) + } +} + +#[derive(Debug)] +pub enum StateType { + Solicit, + Exchange, +} + +#[derive(Debug)] +pub enum EventError { + InvalidEvent(StateType), +} + +#[derive(Debug)] +pub enum EventResponse { + Success, + Prefixes(Vec), +} + +#[derive(Error, Debug)] +pub enum SmError { + #[error("io error: {0}")] + Io(#[from] std::io::Error), + + #[error("discovery error: {0}")] + Discovery(#[from] discovery::DiscoveryError), +} + +#[derive(Clone)] +pub struct Config { + /// Interface this state machine is associated with. + pub if_index: u32, + + /// Interface name this state machine is associated with. + pub if_name: String, + + /// Address object name the state machine uses for peering. Must correspond + /// to IPv6 link local address. + pub aobj_name: String, + + /// Link local Ipv6 address this state machine is associated with + pub addr: Ipv6Addr, + + /// How long to wait between solicitations (milliseconds). + pub solicit_interval: u64, + + /// How often to check for link failure while waiting for discovery messges. + pub discovery_read_timeout: u64, + + /// How long to wait between attempts to get an IP address for a specified + /// address object. + pub ip_addr_wait: u64, + + /// How long to wait without a solicitation response before expiring a peer + /// (milliseconds). + pub expire_threshold: u64, + + /// How long to wait for a response to exchange messages. + pub exchange_timeout: u64, + + /// The kind of router this is, server or transit. + pub kind: RouterKind, + + /// TCP port to use for prefix exchange. + pub exchange_port: u16, + + /// Dendrite dpd config + pub dpd: Option, +} + +#[derive(Clone)] +pub struct DpdConfig { + pub host: String, + pub port: u16, +} + +#[derive(Default)] +pub struct SessionStats { + // Discovery + pub solicitations_sent: AtomicU64, + pub solicitations_received: AtomicU64, + pub advertisements_sent: AtomicU64, + pub advertisements_received: AtomicU64, + pub peer_expirations: AtomicU64, + pub peer_address_changes: AtomicU64, + pub peer_established: AtomicU64, + pub peer_address: Mutex>, + + // Exchange + pub updates_sent: AtomicU64, + pub updates_received: AtomicU64, + pub imported_underlay_prefixes: AtomicU64, + pub imported_tunnel_endpoints: AtomicU64, + pub update_send_fail: AtomicU64, +} + +#[derive(Clone)] +pub struct SmContext { + pub config: Config, + pub db: Db, + pub tx: Sender, + pub event_channels: Vec>, + pub rt: Arc, + pub hostname: String, + pub stats: Arc, + pub log: Logger, +} + +pub struct StateMachine { + pub ctx: SmContext, + pub rx: Option>, +} diff --git a/ddm/src/sm.rs b/ddm/src/sm/state.rs similarity index 88% rename from ddm/src/sm.rs rename to ddm/src/sm/state.rs index 604b90802..d60396894 100644 --- a/ddm/src/sm.rs +++ b/ddm/src/sm/state.rs @@ -2,196 +2,32 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use crate::db::Db; -use crate::discovery::Version; +//! Routing state machine implementation. The `Init` -> `Solicit` -> +//! `Exchange` lifecycle drives kernel route programming via [`crate::sys`] +//! and reads interface addressing through `libnet`. This module is +//! illumos-only. + +use super::{ + AdminEvent, Event, NeighborEvent, PeerEvent, PrefixSet, SmContext, SmError, + StateMachine, +}; use crate::exchange::{MulticastUpdate, TunnelUpdate, UnderlayUpdate, Update}; use crate::{dbg, discovery, err, exchange, inf, wrn}; use ddm_types::db::RouterKind; -use ddm_types::exchange::MulticastPathHop; -use ddm_types::exchange::PathVector; +use ddm_types::exchange::{MulticastPathHop, PathVector}; use libnet::get_ipaddr_info; -use mg_common::net::{MulticastOrigin, TunnelOrigin}; -use oxnet::Ipv6Net; use slog::Logger; use std::collections::HashSet; -use std::net::{IpAddr, Ipv6Addr}; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::mpsc::{Receiver, Sender}; -use std::sync::{Arc, Mutex}; -use std::thread::sleep; -use std::thread::spawn; +use std::net::IpAddr; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc::Receiver; +use std::thread::{sleep, spawn}; use std::time::Duration; -use thiserror::Error; - -#[derive(Debug)] -pub enum AdminEvent { - /// Announce a set of IPv6 prefixes - Announce(PrefixSet), - - /// Withdraw a set of IPv6 prefixes - Withdraw(PrefixSet), - - /// Expire the peer at the specified address - Expire(Ipv6Addr), - - /// Synchronize with active peers by pulling their prefixes. - Sync, -} - -#[derive(Debug)] -pub enum PrefixSet { - Underlay(HashSet), - Tunnel(HashSet), - Multicast(HashSet), -} - -#[derive(Debug)] -pub enum PeerEvent { - Push(Arc), -} - -#[derive(Debug)] -pub enum NeighborEvent { - Advertise((Ipv6Addr, Version)), - SolicitFail, - Expire, -} - -#[derive(Debug)] -pub enum Event { - Neighbor(NeighborEvent), - Peer(PeerEvent), - Admin(AdminEvent), -} -impl From for Event { - fn from(e: NeighborEvent) -> Self { - Self::Neighbor(e) - } -} - -impl From for Event { - fn from(e: PeerEvent) -> Self { - Self::Peer(e) - } -} - -impl From for Event { - fn from(e: AdminEvent) -> Self { - Self::Admin(e) - } -} - -#[derive(Debug)] -pub enum StateType { - Solicit, - Exchange, -} - -#[derive(Debug)] -pub enum EventError { - InvalidEvent(StateType), -} - -#[derive(Debug)] -pub enum EventResponse { - Success, - Prefixes(Vec), -} - -#[derive(Error, Debug)] -pub enum SmError { - #[error("io error: {0}")] - Io(#[from] std::io::Error), - - #[error("discovery error: {0}")] - Discovery(#[from] discovery::DiscoveryError), -} - -#[derive(Clone)] -pub struct Config { - /// Interface this state machine is associated with. - pub if_index: u32, - - /// Interface name this state machine is associated with. - pub if_name: String, - - /// Address object name the state machine uses for peering. Must correspond - /// to IPv6 link local address. - pub aobj_name: String, - - /// Link local Ipv6 address this state machine is associated with - pub addr: Ipv6Addr, - - /// How long to wait between solicitations (milliseconds). - pub solicit_interval: u64, - - /// How often to check for link failure while waiting for discovery messges. - pub discovery_read_timeout: u64, - - /// How long to wait between attempts to get an IP address for a specified - /// address object. - pub ip_addr_wait: u64, - - /// How long to wait without a solicitation response before expiring a peer - /// (milliseconds). - pub expire_threshold: u64, - - /// How long to wait for a response to exchange messages. - pub exchange_timeout: u64, - - /// The kind of router this is, server or transit. - pub kind: RouterKind, - - /// TCP port to use for prefix exchange. - pub exchange_port: u16, - - /// Dendrite dpd config - pub dpd: Option, -} - -#[derive(Clone)] -pub struct DpdConfig { - pub host: String, - pub port: u16, -} - -#[derive(Default)] -pub struct SessionStats { - // Discovery - pub solicitations_sent: AtomicU64, - pub solicitations_received: AtomicU64, - pub advertisements_sent: AtomicU64, - pub advertisements_received: AtomicU64, - pub peer_expirations: AtomicU64, - pub peer_address_changes: AtomicU64, - pub peer_established: AtomicU64, - pub peer_address: Mutex>, - - // Exchange - pub updates_sent: AtomicU64, - pub updates_received: AtomicU64, - pub imported_underlay_prefixes: AtomicU64, - pub imported_tunnel_endpoints: AtomicU64, - pub update_send_fail: AtomicU64, -} - -#[derive(Clone)] -pub struct SmContext { - pub config: Config, - pub db: Db, - pub tx: Sender, - pub event_channels: Vec>, - pub rt: Arc, - pub hostname: String, - pub stats: Arc, - pub log: Logger, -} - -pub struct StateMachine { - pub ctx: SmContext, - pub rx: Option>, -} +use crate::discovery::Version; +use mg_common::net::TunnelOrigin; +use std::net::Ipv6Addr; impl StateMachine { pub fn run(&mut self) -> Result<(), SmError> { @@ -371,8 +207,8 @@ impl State for Solicit { } } -pub struct Exchange { - pub peer: Ipv6Addr, +struct Exchange { + peer: Ipv6Addr, version: Version, ctx: SmContext, log: Logger, @@ -619,7 +455,6 @@ impl State for Exchange { "announce: {}", e, ); - wrn!( self.log, self.ctx.config.if_name, diff --git a/ddm/src/util.rs b/ddm/src/util.rs deleted file mode 100644 index f3a96c03a..000000000 --- a/ddm/src/util.rs +++ /dev/null @@ -1,14 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use std::mem::MaybeUninit; - -//TODO trade for `MaybeUninit::slice_assume_init_ref` when it becomes available -//in stable Rust. -#[inline(always)] -pub(crate) const unsafe fn u8_slice_assume_init_ref( - slice: &[MaybeUninit], -) -> &[u8] { - unsafe { &*(slice as *const [MaybeUninit] as *const [u8]) } -} diff --git a/ddmd/Cargo.toml b/ddmd/Cargo.toml index 5ec0804b0..a0f327c6f 100644 --- a/ddmd/Cargo.toml +++ b/ddmd/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" edition = "2024" [dependencies] -ddm = { path = "../ddm" } +ddm = { path = "../ddm", default-features = false } mg-common = { path = "../mg-common" } anyhow.workspace = true clap.workspace = true @@ -19,3 +19,7 @@ dpd-client.workspace = true anstyle.workspace = true uuid.workspace = true smf.workspace = true + +[features] +default = ["illumos"] +illumos = ["ddm/illumos"] diff --git a/ddmd/src/main.rs b/ddmd/src/main.rs index d5db3f3ca..71dd84400 100644 --- a/ddmd/src/main.rs +++ b/ddmd/src/main.rs @@ -6,11 +6,13 @@ use clap::Parser; use ddm::admin::{HandlerContext, RouterStats}; use ddm::db::Db; use ddm::sm::{DpdConfig, SmContext, StateMachine}; +#[cfg(all(feature = "illumos", target_os = "illumos"))] use ddm::sys::Route; use ddm_types::db::RouterKind; use signal::handle_signals; -use slog::{Drain, Logger, error}; +use slog::{Drain, Logger, error, warn}; use std::net::{IpAddr, Ipv6Addr}; +#[cfg(all(feature = "illumos", target_os = "illumos"))] use std::sync::mpsc::channel; use std::sync::{Arc, Mutex}; use uuid::Uuid; @@ -100,6 +102,15 @@ struct Arg { /// Id of the sled this router is running on. #[arg(long)] sled_uuid: Option, + + /// Skip the routing state machine (discovery, exchange, route + /// synchronization). Only the admin API server runs, allowing test + /// fixtures to obtain a real ddmd admin endpoint without the kernel-level + /// networking the state machine requires. + /// + /// Analogous to `mgd --no-bgp-dispatcher`. + #[arg(long, default_value_t = false)] + no_state_machine: bool, } #[derive(Debug, Parser, Clone)] @@ -121,11 +132,8 @@ async fn run() { .await .expect("set up refresh signal handler"); - let mut event_channels = Vec::new(); let db = Db::new(&format!("{}/ddmdb", arg.data_dir), log.clone()).unwrap(); - let mut sms = Vec::new(); - let dpd = match arg.dendrite { true => Some(DpdConfig { host: arg.dpd_host.clone(), @@ -140,54 +148,10 @@ async fn run() { .to_string_lossy() .to_string(); - for name in arg.addresses { - let (tx, rx) = channel(); - let config = ddm::sm::Config { - solicit_interval: arg.solicit_interval, - expire_threshold: arg.expire_threshold, - discovery_read_timeout: arg.discovery_read_timeout, - ip_addr_wait: arg.ip_addr_wait, - exchange_timeout: arg.exchange_timeout, - exchange_port: arg.exchange_port, - aobj_name: name.clone(), - if_name: String::new(), - if_index: 0, - kind: arg.kind, - dpd: dpd.clone(), - addr: Ipv6Addr::UNSPECIFIED, - }; - let ctx = SmContext { - config, - db: db.clone(), - event_channels: Vec::new(), - tx: tx.clone(), - log: log.clone(), - hostname: hostname.clone(), - rt: rt.clone(), - stats: Arc::new(ddm::sm::SessionStats::default()), - }; - let sm = StateMachine { ctx, rx: Some(rx) }; - sms.push(sm); - event_channels.push(tx); - } - - // Add an event channel sender for each state machine to every other state - // machine. - for (i, sm) in sms.iter_mut().enumerate() { - for (j, e) in event_channels.iter().enumerate() { - // dont give a state machine an event sender to itself. - if i == j { - continue; - } - sm.ctx.event_channels.push(e.clone()); - } - } + let (sms, event_channels) = + start_state_machines(&arg, &db, &dpd, &hostname, &rt, &log); - for sm in &mut sms { - sm.run().unwrap(); - } - - termination_handler(db.clone(), dpd, rt, log.clone()); + termination_handler(db.clone(), dpd.clone(), rt.clone(), log.clone()); let router_stats = Arc::new(RouterStats::default()); let peers: Vec = sms.iter().map(|x| x.ctx.clone()).collect(); @@ -237,6 +201,128 @@ async fn run() { std::thread::park(); } +/// Build, wire, and start the per-address routing state machines. +/// +/// Returns the running [`StateMachine`] handles plus the sender side of each +/// machine's event channel. When `--no-state-machine` is set the function +/// short-circuits to empty vectors, leaving the daemon to serve only its +/// admin API. The illumos and non-illumos variants share that early-exit +/// branch; only the actual machine setup is platform-specific. +#[cfg(all(feature = "illumos", target_os = "illumos"))] +fn start_state_machines( + arg: &Arg, + db: &Db, + dpd: &Option, + hostname: &str, + rt: &Arc, + log: &Logger, +) -> ( + Vec, + Vec>, +) { + if arg.no_state_machine { + if !arg.addresses.is_empty() { + warn!( + log, + "--no-state-machine set; ignoring {} --addr value(s)", + arg.addresses.len(), + ); + } + return (Vec::new(), Vec::new()); + } + + let mut sms = Vec::new(); + let mut event_channels = Vec::new(); + + for name in &arg.addresses { + let (tx, rx) = channel(); + + let config = ddm::sm::Config { + solicit_interval: arg.solicit_interval, + expire_threshold: arg.expire_threshold, + discovery_read_timeout: arg.discovery_read_timeout, + ip_addr_wait: arg.ip_addr_wait, + exchange_timeout: arg.exchange_timeout, + exchange_port: arg.exchange_port, + aobj_name: name.clone(), + if_name: String::new(), + if_index: 0, + kind: arg.kind, + dpd: dpd.clone(), + addr: Ipv6Addr::UNSPECIFIED, + }; + + let ctx = SmContext { + config, + db: db.clone(), + event_channels: Vec::new(), + tx: tx.clone(), + log: log.clone(), + hostname: hostname.to_string(), + rt: rt.clone(), + stats: Arc::new(ddm::sm::SessionStats::default()), + }; + + let sm = StateMachine { ctx, rx: Some(rx) }; + sms.push(sm); + event_channels.push(tx); + } + + // Add an event channel sender for each state machine to every other state + // machine. + for (i, sm) in sms.iter_mut().enumerate() { + for (j, e) in event_channels.iter().enumerate() { + // dont give a state machine an event sender to itself. + if i == j { + continue; + } + sm.ctx.event_channels.push(e.clone()); + } + } + + for sm in &mut sms { + sm.run().unwrap(); + } + + (sms, event_channels) +} + +/// Non-illumos variant: the routing state machine depends on illumos +/// kernel networking, so on every other platform the function logs a warning +/// and returns empty vectors. Test fixtures should pass `--no-state-machine` +/// to silence the warning. +#[cfg(not(all(feature = "illumos", target_os = "illumos")))] +fn start_state_machines( + arg: &Arg, + _db: &Db, + _dpd: &Option, + _hostname: &str, + _rt: &Arc, + log: &Logger, +) -> ( + Vec, + Vec>, +) { + if !arg.no_state_machine { + warn!( + log, + "routing state machine is not available on non-illumos builds; \ + behaving as if `--no-state-machine` were set", + ); + } + if !arg.addresses.is_empty() { + warn!( + log, + "--no-state-machine set; ignoring {} --addr value(s)", + arg.addresses.len(), + ); + } + (Vec::new(), Vec::new()) +} + +/// Install a Ctrl-C handler that withdraws ddmd's imported routes from the +/// kernel before exiting. illumos-only. +#[cfg(all(feature = "illumos", target_os = "illumos"))] fn termination_handler( db: Db, dendrite: Option, @@ -271,6 +357,24 @@ fn termination_handler( }); } +/// Non-illumos variant: there are no kernel routes to withdraw on these +/// platforms, so the handler installs a Ctrl-C task that just exits cleanly. +#[cfg(not(all(feature = "illumos", target_os = "illumos")))] +fn termination_handler( + _db: Db, + _dendrite: Option, + _rt: Arc, + _log: Logger, +) { + tokio::spawn(async { + tokio::signal::ctrl_c() + .await + .expect("error setting termination handler"); + const SIGTERM_EXIT: i32 = 130; + std::process::exit(SIGTERM_EXIT); + }); +} + pub(crate) fn init_logger() -> Logger { let drain = slog_bunyan::new(std::io::stdout()).build().fuse(); let drain = slog_async::Async::new(drain) From fab9beb72c9dd6780740d174733c08ff1e7bfb09 Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Fri, 8 May 2026 06:13:23 +0000 Subject: [PATCH 08/16] [ddm-api] PUT /peer for state-machine-bypassing peer injection Fills out the testing work for `ddmd` in Omicron. Includes: - Adds a `PUT /peer` admin endpoint, gated at v2.0.0 (MULTICAST_SUPPORT), that injects a `PeerInfo` directly into the in-memory peer table at a supplied interface index. Intended for fixtures running ddmd with `--no-state-machine`. Note that in typical operations the discovery handler will overwrite any directly-injected entry the next time a peer is observed on that interface. - Extracts the `get_peers` and `put_peer` impl bodies into `do_get_peers` / `do_put_peer` free functions in `ddm/src/admin.rs`, mirroring `mgd::bgp_admin::do_bgp_apply`, so the endpoints can be exercised in-process. - Adds `tests::put_peer_round_trips` over a tempdir-backed setup covering the round-trip and same interface-index overwrite invariant. - Adds a `put_peer` synthetic injection into the `run_trio_tests` smoke test that exercises the full HTTP path through the generated client against a running ddmd. --- Cargo.lock | 1 + ddm-api/src/lib.rs | 17 +++ ddm-types/versions/src/latest.rs | 1 + .../versions/src/multicast_support/admin.rs | 16 +++ .../versions/src/multicast_support/mod.rs | 1 + ddm/Cargo.toml | 1 + ddm/src/admin.rs | 105 +++++++++++++++++- ...aeda2.json => ddm-admin-2.0.0-0cfd90.json} | 46 ++++++++ openapi/ddm-admin/ddm-admin-latest.json | 2 +- tests/src/ddm.rs | 30 ++++- 10 files changed, 215 insertions(+), 5 deletions(-) create mode 100644 ddm-types/versions/src/multicast_support/admin.rs rename openapi/ddm-admin/{ddm-admin-2.0.0-8aeda2.json => ddm-admin-2.0.0-0cfd90.json} (93%) diff --git a/Cargo.lock b/Cargo.lock index 39d549446..7ced62a8e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1342,6 +1342,7 @@ dependencies = [ "sled", "slog", "socket2", + "tempfile", "thiserror 2.0.18", "tokio", "uuid", diff --git a/ddm-api/src/lib.rs b/ddm-api/src/lib.rs index 6f3f82825..b86622ceb 100644 --- a/ddm-api/src/lib.rs +++ b/ddm-api/src/lib.rs @@ -72,6 +72,23 @@ pub trait DdmAdminApi { params: Path, ) -> Result; + /// Set peer information for a given interface index, bypassing the state machine. + /// + /// Intended for test fixtures that run `ddmd` with `--no-state-machine`. + /// In a normal run, discovery writes peer entries keyed by interface + /// index whenever it processes an advertisement, so any directly-injected + /// entry for an active interface will be overwritten the next time a + /// peer is observed there. + #[endpoint { + method = PUT, + path = "/peer", + versions = VERSION_MULTICAST_SUPPORT.. + }] + async fn put_peer( + ctx: RequestContext, + request: TypedBody, + ) -> Result; + #[endpoint { method = GET, path = "/originated" }] async fn get_originated( ctx: RequestContext, diff --git a/ddm-types/versions/src/latest.rs b/ddm-types/versions/src/latest.rs index 721f8cbf8..dcee5b6f7 100644 --- a/ddm-types/versions/src/latest.rs +++ b/ddm-types/versions/src/latest.rs @@ -8,6 +8,7 @@ pub mod admin { pub use crate::v1::admin::EnableStatsRequest; pub use crate::v1::admin::ExpirePathParams; pub use crate::v1::admin::PrefixMap; + pub use crate::v2::admin::PutPeerRequest; } pub mod db { diff --git a/ddm-types/versions/src/multicast_support/admin.rs b/ddm-types/versions/src/multicast_support/admin.rs new file mode 100644 index 000000000..dc217dc14 --- /dev/null +++ b/ddm-types/versions/src/multicast_support/admin.rs @@ -0,0 +1,16 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use super::db::PeerInfo; + +/// Body for `PUT /peer`. Sets `info` at the slot keyed by `if_index` +/// (interface index) in the in-memory peer map. +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] +pub struct PutPeerRequest { + pub if_index: u32, + pub info: PeerInfo, +} diff --git a/ddm-types/versions/src/multicast_support/mod.rs b/ddm-types/versions/src/multicast_support/mod.rs index 066113e24..9ac0ecc65 100644 --- a/ddm-types/versions/src/multicast_support/mod.rs +++ b/ddm-types/versions/src/multicast_support/mod.rs @@ -5,5 +5,6 @@ //! Types from API version 2 (MULTICAST_SUPPORT) that add multicast //! group management to the DDM admin API. +pub mod admin; pub mod db; pub mod exchange; diff --git a/ddm/Cargo.toml b/ddm/Cargo.toml index 373f90b06..1e7f5a85d 100644 --- a/ddm/Cargo.toml +++ b/ddm/Cargo.toml @@ -5,6 +5,7 @@ edition = "2024" [dev-dependencies] pretty_assertions.workspace = true +tempfile = "3" [dependencies] slog.workspace = true diff --git a/ddm/src/admin.rs b/ddm/src/admin.rs index 9ba23ebf3..1322eb10d 100644 --- a/ddm/src/admin.rs +++ b/ddm/src/admin.rs @@ -6,7 +6,9 @@ use crate::db::Db; use crate::sm::{AdminEvent, Event, PrefixSet, SmContext}; use ddm_api::DdmAdminApi; use ddm_api::ddm_admin_api_mod; -use ddm_types::admin::{EnableStatsRequest, ExpirePathParams, PrefixMap}; +use ddm_types::admin::{ + EnableStatsRequest, ExpirePathParams, PrefixMap, PutPeerRequest, +}; use ddm_types::db::{MulticastRoute, PeerInfo, TunnelRoute}; use ddm_types::exchange::PathVector; use dropshot::ApiDescription; @@ -112,8 +114,7 @@ impl DdmAdminApi for DdmAdminApiImpl { async fn get_peers( ctx: RequestContext, ) -> Result>, HttpError> { - let ctx = lock!(ctx.context()); - Ok(HttpResponseOk(ctx.db.peers())) + Ok(HttpResponseOk(do_get_peers(ctx.context()))) } async fn get_peers_v1( @@ -151,6 +152,14 @@ impl DdmAdminApi for DdmAdminApiImpl { Ok(HttpResponseUpdatedNoContent()) } + async fn put_peer( + ctx: RequestContext, + request: TypedBody, + ) -> Result { + do_put_peer(ctx.context(), request.into_inner()); + Ok(HttpResponseUpdatedNoContent()) + } + async fn get_originated( ctx: RequestContext, ) -> Result>, HttpError> { @@ -481,3 +490,93 @@ pub fn api_description() { ddm_admin_api_mod::api_description::() } + +/// Snapshot the current peer table, keyed by interface index. +pub(crate) fn do_get_peers( + ctx: &Arc>, +) -> HashMap { + let ctx = lock!(ctx); + ctx.db.peers() +} + +/// Insert or replace the peer entry at `request.if_index`. Tests bypass +/// the dropshot endpoint and call this directly; production goes through +/// [`DdmAdminApiImpl::put_peer`]. +pub(crate) fn do_put_peer( + ctx: &Arc>, + request: PutPeerRequest, +) { + let PutPeerRequest { if_index, info } = request; + let ctx = lock!(ctx); + ctx.db.set_peer(if_index, info); +} + +#[cfg(test)] +mod tests { + use super::{HandlerContext, RouterStats, do_get_peers, do_put_peer}; + use crate::db::Db; + use ddm_types::admin::PutPeerRequest; + use ddm_types::db::{PeerInfo, PeerStatus, RouterKind}; + use slog::{Discard, Logger, o}; + use std::sync::{Arc, Mutex}; + use tempfile::TempDir; + + fn build_context(tmpdir: &TempDir) -> Arc> { + let log = Logger::root(Discard, o!()); + let db_path = tmpdir.path().join("ddm").to_str().unwrap().to_string(); + let db = Db::new(&db_path, log.clone()).expect("open db"); + Arc::new(Mutex::new(HandlerContext { + event_channels: vec![], + db, + stats: Arc::new(RouterStats::default()), + peers: vec![], + stats_handler: Arc::new(Mutex::new(None)), + log, + })) + } + + #[test] + fn put_peer_round_trips() { + let tmpdir = TempDir::new().expect("tempdir"); + let ctx = build_context(&tmpdir); + + let info = PeerInfo { + status: PeerStatus::Active, + addr: "fd00::1".parse().unwrap(), + host: "test-sled-1".to_string(), + kind: RouterKind::Server, + if_name: Some("tfportrear0_0".to_string()), + }; + + do_put_peer( + &ctx, + PutPeerRequest { + if_index: 7, + info: info.clone(), + }, + ); + + let peers = do_get_peers(&ctx); + assert_eq!(peers.len(), 1); + let got = peers.get(&7).expect("peer at if_index 7"); + assert_eq!(got, &info); + + // Overwriting at the same `if_index` replaces the entry rather + // than creating a second one. + let info2 = PeerInfo { + addr: "fd00::2".parse().unwrap(), + host: "test-sled-1-replaced".to_string(), + ..info + }; + do_put_peer( + &ctx, + PutPeerRequest { + if_index: 7, + info: info2.clone(), + }, + ); + let peers = do_get_peers(&ctx); + assert_eq!(peers.len(), 1, "overwrite at same if_index keeps map size",); + assert_eq!(peers[&7].addr, info2.addr); + } +} diff --git a/openapi/ddm-admin/ddm-admin-2.0.0-8aeda2.json b/openapi/ddm-admin/ddm-admin-2.0.0-0cfd90.json similarity index 93% rename from openapi/ddm-admin/ddm-admin-2.0.0-8aeda2.json rename to openapi/ddm-admin/ddm-admin-2.0.0-0cfd90.json index 312fadd40..5684230ad 100644 --- a/openapi/ddm-admin/ddm-admin-2.0.0-8aeda2.json +++ b/openapi/ddm-admin/ddm-admin-2.0.0-0cfd90.json @@ -223,6 +223,34 @@ } } }, + "/peer": { + "put": { + "summary": "Set peer information for a given interface index, bypassing the state machine.", + "description": "Intended for test fixtures that run `ddmd` with `--no-state-machine`. In a normal run, discovery writes peer entries keyed by interface index whenever it processes an advertisement, so any directly-injected entry for an active interface will be overwritten the next time a peer is observed there.", + "operationId": "put_peer", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PutPeerRequest" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/peers": { "get": { "operationId": "get_peers", @@ -717,6 +745,24 @@ "Expired" ] }, + "PutPeerRequest": { + "description": "Body for `PUT /peer`. Sets `info` at the slot keyed by `if_index` (interface index) in the in-memory peer map.", + "type": "object", + "properties": { + "if_index": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "info": { + "$ref": "#/components/schemas/PeerInfo" + } + }, + "required": [ + "if_index", + "info" + ] + }, "RouterKind": { "type": "integer", "enum": [ diff --git a/openapi/ddm-admin/ddm-admin-latest.json b/openapi/ddm-admin/ddm-admin-latest.json index aaa8691d3..0032bd2a9 120000 --- a/openapi/ddm-admin/ddm-admin-latest.json +++ b/openapi/ddm-admin/ddm-admin-latest.json @@ -1 +1 @@ -ddm-admin-2.0.0-8aeda2.json \ No newline at end of file +ddm-admin-2.0.0-0cfd90.json \ No newline at end of file diff --git a/tests/src/ddm.rs b/tests/src/ddm.rs index 1d92dc381..fde1f8af5 100644 --- a/tests/src/ddm.rs +++ b/tests/src/ddm.rs @@ -4,7 +4,10 @@ use anyhow::{Result, anyhow}; use ddm_admin_client::Client; -use ddm_admin_client::types::{MulticastOrigin, TunnelOrigin, Vni}; +use ddm_admin_client::types::{ + MulticastOrigin, PeerInfo, PeerStatus, PutPeerRequest, RouterKind, + TunnelOrigin, Vni, +}; use slog::{Drain, Logger}; use std::env; use std::net::Ipv6Addr; @@ -462,6 +465,31 @@ async fn run_trio_tests( println!("initial peering test passed"); + // PUT /peer smoke against a running ddmd. Use an unused interface + // index so the live discovery handler does not race the injection + // on a real interface. + let synthetic = PeerInfo { + status: PeerStatus::Active, + addr: "fd00::dead:beef".parse().unwrap(), + host: "synthetic".to_string(), + // RouterKind is integer-encoded in the generated client schema; + // 0 is `Server`. See ddm-types::initial::db::RouterKind. + kind: RouterKind::try_from(0_i64).unwrap(), + if_name: Some("synthetic0".to_string()), + }; + + t1.put_peer(&PutPeerRequest { + if_index: 9999, + info: synthetic.clone(), + }) + .await?; + + wait_for_eq!(t1.get_peers().await.map_or(99, |x| x.len()), 3); + let peers = t1.get_peers().await?; + assert_eq!(peers["9999"].host, "synthetic"); + + println!("put_peer synthetic injection passed"); + s1.advertise_prefixes(&vec!["fd00:1::/64".parse().unwrap()]) .await?; From 60a5a2495412e4a1d6189eda1b8931588d7c9dbb Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Tue, 2 Jun 2026 13:06:40 +0000 Subject: [PATCH 09/16] mg-common: shared tfport name parser for qsfp and rear ports The tfport datalink naming logic, parsing names like tfportrear0_0 and tfportqsfp0_0 into a kind, port, and link, was duplicated in ddm's sys module and mg-lower's dendrite module. Hoist it into a single mg_common::tfport module so both consumers share one parser and one set of error types. No behavior change. --- Cargo.lock | 2 + ddm/src/sys.rs | 34 +++--- mg-common/Cargo.toml | 2 + mg-common/src/lib.rs | 1 + mg-common/src/tfport.rs | 247 +++++++++++++++++++++++++++++++++++++++ mg-lower/src/dendrite.rs | 94 ++------------- 6 files changed, 278 insertions(+), 102 deletions(-) create mode 100644 mg-common/src/tfport.rs diff --git a/Cargo.lock b/Cargo.lock index 88b18c667..a58c37063 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3793,12 +3793,14 @@ dependencies = [ "backoff", "clap", "ddm-api-types", + "dpd-client", "libc", "libnet", "omicron-common", "oximeter", "oximeter-producer", "oxnet", + "proptest", "schemars 0.8.22", "serde", "serde_json", diff --git a/ddm/src/sys.rs b/ddm/src/sys.rs index c251ab600..d300f8ada 100644 --- a/ddm/src/sys.rs +++ b/ddm/src/sys.rs @@ -8,6 +8,7 @@ use ddm_api_types::db::TunnelRoute; use dpd_client::Client; use dpd_client::ClientState; use dpd_client::types; +use mg_common::tfport::{TfportKind, parse_tfport_name, tfport_port_id}; use oxnet::{IpNet, Ipv4Net, Ipv6Net}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -176,25 +177,25 @@ pub fn add_routes_dendrite( } }; - // TODO this is gross, use link type properties rather than futzing - // around with strings. - let Some(egress_port_num) = ifname - .strip_prefix("tfportrear") - .and_then(|x| x.strip_suffix("_0")) - .map(|x| x.trim()) - .and_then(|x| x.parse::().ok()) - else { - err!(log, ifname, "expected tfportrear"); - continue; + let tfport = match parse_tfport_name(ifname) { + Ok(tfport) => tfport, + Err(e) => { + err!(log, ifname, "{e}"); + continue; + } }; // TODO this assumes ddm only operates on rear ports, which will not be // true for multi-rack deployments. - let port_name = format!("rear{}", egress_port_num); - let port_id = match types::Rear::try_from(&port_name) { - Ok(rear) => PortId::Rear(rear), + if tfport.kind != TfportKind::Rear { + err!(log, ifname, "expected tfportrear"); + continue; + } + + let port_id = match tfport_port_id(tfport.kind, tfport.port) { + Ok(port_id) => port_id, Err(e) => { - err!(log, ifname, "bad port name ({port_name}): {e}"); + err!(log, ifname, "{e}"); continue; } }; @@ -206,11 +207,10 @@ pub fn add_routes_dendrite( r.dest, r.gw, port_id, - 0, + tfport.link, ); - // TODO breakout considerations - let link_id = types::LinkId(0); + let link_id = types::LinkId(tfport.link); let target = types::Ipv6Route { tag: DDM_DPD_TAG.into(), diff --git a/mg-common/Cargo.toml b/mg-common/Cargo.toml index 6e28aa395..262fb77a8 100644 --- a/mg-common/Cargo.toml +++ b/mg-common/Cargo.toml @@ -7,6 +7,7 @@ edition = "2024" clap.workspace = true anyhow.workspace = true ddm-api-types.workspace = true +dpd-client.workspace = true anstyle.workspace = true serde.workspace = true schemars.workspace = true @@ -28,6 +29,7 @@ workspace = true optional = true [dev-dependencies] +proptest.workspace = true serde_json.workspace = true [features] diff --git a/mg-common/src/lib.rs b/mg-common/src/lib.rs index 01c025512..97bf6c353 100644 --- a/mg-common/src/lib.rs +++ b/mg-common/src/lib.rs @@ -9,6 +9,7 @@ pub mod nexus; pub mod smf; pub mod stats; pub mod test; +pub mod tfport; pub mod thread; use std::time::Duration; diff --git a/mg-common/src/tfport.rs b/mg-common/src/tfport.rs new file mode 100644 index 000000000..62379de72 --- /dev/null +++ b/mg-common/src/tfport.rs @@ -0,0 +1,247 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Parsing of Tofino port (`tfport`) datalink names. +//! +//! Tofino switch ports are surfaced to the host as illumos datalinks named +//! `tfport_[.vlan]`; for example, `tfportqsfp10_0` (front +//! panel) or `tfportrear0_0.100` (backplane). This module parses that form +//! into its components so callers can map a datalink name back to a switch +//! port without each one hand-rolling its own string handling. + +/// Prefix shared by every `tfport` datalink name. +const TFPORT_DEVICE_PREFIX: &str = "tfport"; + +/// Switch-port device kind encoded in a `tfport` datalink name. +/// +/// Front-panel links typically appear as `qsfp`. Backplane links toward other +/// sleds (the multicast underlay path) typically appear as `rear`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TfportKind { + Qsfp, + Rear, +} + +impl TfportKind { + /// Device token as it appears both in a tfport name (`tfportrear0_0`) and + /// in a dpd port name (`rear0`). + pub fn token(self) -> &'static str { + match self { + TfportKind::Qsfp => "qsfp", + TfportKind::Rear => "rear", + } + } + + /// Parse a device kind from its datalink token (`qsfp`, `rear`). + pub fn from_token(s: &str) -> Option { + match s { + "qsfp" => Some(TfportKind::Qsfp), + "rear" => Some(TfportKind::Rear), + _ => None, + } + } +} + +/// Components parsed from a `tfport` datalink name. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TfportName { + /// Device kind (front-panel vs backplane). + pub kind: TfportKind, + /// Switch port number within the kind. + pub port: u8, + /// Link (lane) number within the port. + pub link: u8, + /// Optional VLAN tag appended after a `.`. + pub vlan: Option, +} + +/// Parse a `tfport` datalink name into its components. +/// +/// # Examples +/// +/// ``` +/// use mg_common::tfport::{parse_tfport_name, TfportKind, TfportName}; +/// assert_eq!( +/// parse_tfport_name("tfportqsfp10_0.100").unwrap(), +/// TfportName { kind: TfportKind::Qsfp, port: 10, link: 0, vlan: Some(100) }, +/// ); +/// assert_eq!( +/// parse_tfport_name("tfportrear0_0").unwrap(), +/// TfportName { kind: TfportKind::Rear, port: 0, link: 0, vlan: None }, +/// ); +/// ``` +/// +/// # Errors +/// +/// Returns a human-readable message if `name` lacks the `tfport` prefix, has +/// an unrecognized device kind, or has malformed port/link/vlan fields. +pub fn parse_tfport_name(name: &str) -> Result { + let body = name.strip_prefix(TFPORT_DEVICE_PREFIX).ok_or_else(|| { + format!("{name} missing expected prefix {TFPORT_DEVICE_PREFIX}") + })?; + + // The device kind is the leading alphabetic run (`qsfp`, `rear`), the + // remainder carries the port/link/vlan numbers. + let split = body + .find(|c: char| !c.is_ascii_alphabetic()) + .ok_or_else(|| format!("{name} has no port id"))?; + let (kind_str, rest) = body.split_at(split); + let kind = TfportKind::from_token(kind_str).ok_or_else(|| { + format!("{name} has unsupported device kind {kind_str}") + })?; + + let (port_link, vlan_str) = match rest.split_once('.') { + Some((port_link, vlan)) => (port_link, Some(vlan)), + None => (rest, None), + }; + + let (port, link) = port_link + .split_once('_') + .ok_or_else(|| format!("{name} has no link id"))?; + + let port = port + .parse::() + .map_err(|_| format!("{name} has invalid port {port}"))?; + + let link = link + .parse::() + .map_err(|_| format!("{name} has invalid link id {link}"))?; + + let vlan = match vlan_str { + None => None, + // A second `.` (e.g. `tfportqsfp10_0.100.200`) leaves a non-numeric + // remainder, so the parse below rejects it. + Some(vlan) => Some( + vlan.parse::() + .map_err(|_| format!("{name} has invalid vlan {vlan}"))?, + ), + }; + + Ok(TfportName { + kind, + port, + link, + vlan, + }) +} + +/// Build a dpd [`PortId`] from a parsed tfport kind and port number. +/// +/// # Errors +/// +/// Returns a human-readable message if the synthesized port name is not a valid +/// dpd `qsfp` or `rear` port identifier. +/// +/// [`PortId`]: dpd_client::types::PortId +pub fn tfport_port_id( + kind: TfportKind, + port: u8, +) -> Result { + use dpd_client::types; + + let port_name = format!("{}{}", kind.token(), port); + match kind { + TfportKind::Qsfp => types::Qsfp::try_from(&port_name) + .map(types::PortId::Qsfp) + .map_err(|e| format!("bad qsfp port name {port_name}: {e}")), + TfportKind::Rear => types::Rear::try_from(&port_name) + .map(types::PortId::Rear) + .map_err(|e| format!("bad rear port name {port_name}: {e}")), + } +} + +#[cfg(test)] +mod tests { + use super::TfportKind::{Qsfp, Rear}; + use super::{TfportName, parse_tfport_name}; + use proptest::prelude::*; + + fn name( + kind: super::TfportKind, + port: u8, + link: u8, + vlan: Option, + ) -> TfportName { + TfportName { + kind, + port, + link, + vlan, + } + } + + #[test] + fn test_tfport_parser() { + // Valid qsfp (front-panel) names. + assert_eq!( + parse_tfport_name("tfportqsfp10_0").unwrap(), + name(Qsfp, 10, 0, None) + ); + assert_eq!( + parse_tfport_name("tfportqsfp10_0.100").unwrap(), + name(Qsfp, 10, 0, Some(100)) + ); + assert_eq!( + parse_tfport_name("tfportqsfp1_1").unwrap(), + name(Qsfp, 1, 1, None) + ); + + // Valid rear (backplane) names. + assert_eq!( + parse_tfport_name("tfportrear0_0").unwrap(), + name(Rear, 0, 0, None) + ); + assert_eq!( + parse_tfport_name("tfportrear31_0.200").unwrap(), + name(Rear, 31, 0, Some(200)) + ); + + // Malformed names. + assert!(parse_tfport_name("fportqsfp10_0").is_err()); + assert!(parse_tfport_name("10_0").is_err()); + assert!(parse_tfport_name("tfportqsfp10").is_err()); + assert!(parse_tfport_name("tfportqsfp_10").is_err()); + assert!(parse_tfport_name("tfportqsfp0_").is_err()); + assert!(parse_tfport_name("tfportqsfp10_10_10").is_err()); + assert!(parse_tfport_name("tfportqsfp10.100_0").is_err()); + + // Unsupported or missing device kind. + assert!(parse_tfport_name("tfportfoo0_0").is_err()); + assert!(parse_tfport_name("tfport0_0").is_err()); + + // Invalid numeric components. + assert!(parse_tfport_name("tfportqsfp1X_0.100").is_err()); + assert!(parse_tfport_name("tfportqsfp10_X.100").is_err()); + assert!(parse_tfport_name("tfportqsfp10_0.X").is_err()); + } + + proptest! { + /// Any well-formed name round-trips: formatting a kind, port, link, + /// vlan tuple and parsing it back yields the same components. The + /// parser is purely syntactic, so the full u8/u16 ranges are exercised. + #[test] + fn prop_roundtrip( + is_rear in any::(), + port in any::(), + link in any::(), + vlan in proptest::option::of(any::()), + ) { + let kind = if is_rear { Rear } else { Qsfp }; + let mut ifname = format!("tfport{}{port}_{link}", kind.token()); + if let Some(vlan) = vlan { + ifname.push_str(&format!(".{vlan}")); + } + prop_assert_eq!( + parse_tfport_name(&ifname).unwrap(), + TfportName { kind, port, link, vlan }, + ); + } + + /// Parsing arbitrary input never panics; it always returns a `Result`. + #[test] + fn prop_never_panics(ifname in ".*") { + let _ = parse_tfport_name(&ifname); + } + } +} diff --git a/mg-lower/src/dendrite.rs b/mg-lower/src/dendrite.rs index 14ecd375a..c225d11cf 100644 --- a/mg-lower/src/dendrite.rs +++ b/mg-lower/src/dendrite.rs @@ -12,6 +12,7 @@ use dpd_client::Client as DpdClient; use dpd_client::types::{self, LinkState, Route}; use mg_api_types::rdb::path::Path; use mg_api_types::rdb::prefix::Prefix; +use mg_common::tfport::{parse_tfport_name, tfport_port_id}; use oxnet::{IpNet, Ipv4Net, Ipv6Net}; use slog::Logger; use std::{ @@ -22,7 +23,6 @@ use std::{ time::Duration, }; -const TFPORT_QSFP_DEVICE_PREFIX: &str = "tfportqsfp"; const UNIT_DPD: &str = "dpd"; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] @@ -335,70 +335,6 @@ where Ok(()) } -// Translate a tfport name into the underlying (port, link, vlan) tuple. -// tfportqsfp10_0 would translate to (10, 0, None) -// tfportqsfp10_0.100 would translate to (10, 0, Some(100)) -// TODO this is gross, use link type properties rather than futzing -// around with strings. -fn parse_tfport_name(name: &str) -> Result<(u8, u8, Option), Error> { - let body = - name.strip_prefix(TFPORT_QSFP_DEVICE_PREFIX) - .ok_or(Error::Tfport(format!( - "{} missing expected prefix {}", - name, TFPORT_QSFP_DEVICE_PREFIX - )))?; - let fields: Vec<&str> = body.split('.').collect(); - let (port, link) = fields[0] - .split_once('_') - .ok_or(Error::Tfport(format!("{} has no link id", name)))?; - - let port = port.parse::().map_err(|_| { - Error::Tfport(format!("{} has invalid port {}", name, port)) - })?; - - let link = link.parse::().map_err(|_| { - Error::Tfport(format!("{} has invalid link id {}", name, link)) - })?; - - let vlan = match fields.len() { - 1 => Ok(None), - 2 => fields[1].parse::().map(Some).map_err(|_| { - Error::Tfport(format!("{} has invalid vlan {}", name, fields[1])) - }), - _ => Err(Error::Tfport(format!( - "{} has multiple vlan deliminators", - name - ))), - }?; - - Ok((port, link, vlan)) -} - -#[test] -fn test_tfport_parser() { - // Test valid names - assert_eq!(parse_tfport_name("tfportqsfp10_0").unwrap(), (10, 0, None)); - assert_eq!( - parse_tfport_name("tfportqsfp10_0.100").unwrap(), - (10, 0, Some(100)) - ); - assert_eq!(parse_tfport_name("tfportqsfp1_1").unwrap(), (1, 1, None)); - - // test malformed names - assert!(parse_tfport_name("fportqsfp10_0").is_err()); - assert!(parse_tfport_name("10_0").is_err()); - assert!(parse_tfport_name("tfportqsfp10").is_err()); - assert!(parse_tfport_name("tfportqsfp_10").is_err()); - assert!(parse_tfport_name("tfportqsfp0_").is_err()); - assert!(parse_tfport_name("tfportqsfp10_10_10").is_err()); - assert!(parse_tfport_name("tfportqsfp10.100_0").is_err()); - - // test invalid components - assert!(parse_tfport_name("tfportqsfp1X_0.100").is_err()); - assert!(parse_tfport_name("tfportqsfp10_X.100").is_err()); - assert!(parse_tfport_name("tfportqsfp10_0.X").is_err()); -} - fn get_port_and_link( sw: &impl SwitchZone, path: &Path, @@ -408,17 +344,11 @@ fn get_port_and_link( && nh6.is_unicast_link_local() && let Some(ref iface) = path.nexthop_interface { - let (port, link, _vlan) = parse_tfport_name(iface)?; - let port_name = format!("qsfp{port}"); - let port_id = types::Qsfp::try_from(&port_name) - .map(types::PortId::Qsfp) - .map_err(|e| { - Error::Tfport(format!( - "bad port name ifname: {iface} port name: {port_name}: {e}", - )) - })?; + let tfport = parse_tfport_name(iface).map_err(Error::Tfport)?; + let port_id = + tfport_port_id(tfport.kind, tfport.port).map_err(Error::Tfport)?; // TODO breakout considerations - let link_id = types::LinkId(link); + let link_id = types::LinkId(tfport.link); return Ok((port_id, link_id)); } @@ -443,18 +373,12 @@ fn resolve_port_and_link( } }; - let (port, link, _vlan) = parse_tfport_name(&ifname)?; - let port_name = format!("qsfp{port}"); - let port_id = types::Qsfp::try_from(&port_name) - .map(types::PortId::Qsfp) - .map_err(|e| { - Error::Tfport(format!( - "bad port name ifname: {ifname} port name: {port_name}: {e}" - )) - })?; + let tfport = parse_tfport_name(&ifname).map_err(Error::Tfport)?; + let port_id = + tfport_port_id(tfport.kind, tfport.port).map_err(Error::Tfport)?; // TODO breakout considerations - let link_id = types::LinkId(link); + let link_id = types::LinkId(tfport.link); Ok((port_id, link_id)) } From acb26b96ac85f29799e29097ddaab86ff6f9fdf4 Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Tue, 2 Jun 2026 17:02:30 +0000 Subject: [PATCH 10/16] [refactor] mg-lower: extract `port_link_from_ifname` helper from dendrite mod The call sites that resolve a tfport interface name to a DPD (PortId, LinkId) tuple repeated the same inline parse-and-construct sequence. This work factors it into a single `port_link_from_ifname` helper function so that the resolution lives in one place. Note: no behavior change. --- mg-lower/src/dendrite.rs | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/mg-lower/src/dendrite.rs b/mg-lower/src/dendrite.rs index c225d11cf..da7e1ab11 100644 --- a/mg-lower/src/dendrite.rs +++ b/mg-lower/src/dendrite.rs @@ -335,6 +335,19 @@ where Ok(()) } +/// Resolve a tfport datalink name (e.g. `tfportrear0_0`) to the dpd +/// `(PortId, LinkId)` pair that names the switch port and link. +pub(crate) fn port_link_from_ifname( + ifname: &str, +) -> Result<(types::PortId, types::LinkId), Error> { + let tfport = parse_tfport_name(ifname).map_err(Error::Tfport)?; + let port_id = + tfport_port_id(tfport.kind, tfport.port).map_err(Error::Tfport)?; + // TODO breakout considerations + let link_id = types::LinkId(tfport.link); + Ok((port_id, link_id)) +} + fn get_port_and_link( sw: &impl SwitchZone, path: &Path, @@ -344,12 +357,7 @@ fn get_port_and_link( && nh6.is_unicast_link_local() && let Some(ref iface) = path.nexthop_interface { - let tfport = parse_tfport_name(iface).map_err(Error::Tfport)?; - let port_id = - tfport_port_id(tfport.kind, tfport.port).map_err(Error::Tfport)?; - // TODO breakout considerations - let link_id = types::LinkId(tfport.link); - return Ok((port_id, link_id)); + return port_link_from_ifname(iface); } // Standard nexthop resolution for numbered peers @@ -373,13 +381,7 @@ fn resolve_port_and_link( } }; - let tfport = parse_tfport_name(&ifname).map_err(Error::Tfport)?; - let port_id = - tfport_port_id(tfport.kind, tfport.port).map_err(Error::Tfport)?; - - // TODO breakout considerations - let link_id = types::LinkId(tfport.link); - Ok((port_id, link_id)) + port_link_from_ifname(&ifname) } pub(crate) fn get_routes_for_prefix( From e5060279be915839ed63498c788d4a1ff2b00b77 Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Wed, 3 Jun 2026 14:08:54 +0000 Subject: [PATCH 11/16] [design-change] ddm: program DPD multicast members in ddmd from imported routes This work splits DDM underlay multicast (RFD 488) into two directional halves. Multicast origination advertises locally originated MRIB groups outward (MRIB to the DDM admin API to the underlay). Membership, in turn, consumes the routes other sleds originate and programs the local switch's replication members in DPD directly. This commit implements the membership half and refactors origination to match the unicast lower-half's shape and approach. Membership updates run in `ddmd` because both of its inputs (the set of DDM-imported multicast routes and the peer table) are already owned in-process by the DDM state machine. A dedicated sweep thread (`ddm::mcast::run`) reconciles each affected underlay group's DPD members. This mechanism is trigger-driven over an mpsc channel from the import, withdraw, peer-expiry, and peer-link-resolution paths, with a 10s backstop and a startup seed from DPD's member-bearing groups so orphans left by a withdraw during downtime are properly drained. Each DPD member GET/PUT is bounded at 3s (DPD_REQUEST_TIMEOUT, kept so a group's fetch-write pair stays under the chosen backstop interval) and stalls are classified as typed `TimedOut` outcomes. Therefore, an unresponsive DPD retains the group for the next pass rather than dropping it and leaking any replication state. mg-lower's mrib module is now origination-only, watch-driven with a 1s periodic resync backstop, mirroring the unicast lower-half loop. The DPD and DDM clients take an optional address so integration harnesses can target a listener on a dynamically assigned port, exposed as the --dendrite-addr and --ddm-addr mgd flags respectively. Supporting changes attached to this work: - db: factor the imported-route tree scan into scan_origin_tree and return removed unicast and multicast next hops together via `RemovedNexthopRoutes`. - ddmd: wire the sweep's notify channel and start it unconditionally, including --api-only, where it owns the receiver with a known empty peer set. - admin: collect get_peers via filter_map rather than a manual loop. --- Cargo.lock | 214 +-- Cargo.toml | 3 +- ddm-api-types/versions/Cargo.toml | 4 +- .../versions/src/multicast_support/db.rs | 14 +- .../src/multicast_support/exchange.rs | 2 +- .../versions/src/multicast_support/net.rs | 53 +- ddm/Cargo.toml | 4 +- ddm/src/admin.rs | 45 +- ddm/src/db.rs | 179 ++- ddm/src/exchange/mod.rs | 22 +- ddm/src/exchange/runtime.rs | 78 +- ddm/src/lib.rs | 8 +- ddm/src/mcast.rs | 1184 +++++++++++++++++ ddm/src/sm/mod.rs | 5 + ddm/src/sm/state.rs | 55 +- ddm/src/sys.rs | 6 +- ddmd/src/main.rs | 65 +- mg-common/src/net.rs | 2 +- mg-common/src/tfport.rs | 56 +- mg-lower/src/ddm.rs | 19 +- mg-lower/src/dendrite.rs | 29 +- mg-lower/src/mrib.rs | 326 ++++- mg-lower/src/platform.rs | 4 +- mgd/src/main.rs | 62 +- multicast-types/src/lib.rs | 15 - 25 files changed, 1976 insertions(+), 478 deletions(-) create mode 100644 ddm/src/mcast.rs diff --git a/Cargo.lock b/Cargo.lock index a58c37063..9a457deb3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -868,7 +868,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -884,7 +884,7 @@ dependencies = [ [[package]] name = "common" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/dendrite?rev=37992295b5dc708d8f120cee805d67418741b556#37992295b5dc708d8f120cee805d67418741b556" +source = "git+https://github.com/oxidecomputer/dendrite?rev=f27f612dde317f2f05716c761a14500e46581ff7#f27f612dde317f2f05716c761a14500e46581ff7" dependencies = [ "anyhow", "chrono", @@ -1348,6 +1348,7 @@ dependencies = [ "ddm-api-types-versions", "dpd-client", "dropshot", + "futures", "hostname 0.4.2", "http-body-util", "hyper", @@ -1362,6 +1363,7 @@ dependencies = [ "oximeter-producer", "oxnet", "pretty_assertions", + "reqwest 0.13.3", "schemars 0.8.22", "serde", "serde_json", @@ -1623,7 +1625,7 @@ dependencies = [ [[package]] name = "dpd-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/dendrite?rev=37992295b5dc708d8f120cee805d67418741b556#37992295b5dc708d8f120cee805d67418741b556" +source = "git+https://github.com/oxidecomputer/dendrite?rev=f27f612dde317f2f05716c761a14500e46581ff7#f27f612dde317f2f05716c761a14500e46581ff7" dependencies = [ "async-trait", "chrono", @@ -1632,9 +1634,9 @@ dependencies = [ "futures", "http", "oxnet", - "progenitor 0.11.2", + "progenitor 0.13.0", "regress 0.10.5", - "reqwest 0.12.28", + "reqwest 0.13.3", "schemars 0.8.22", "serde", "serde_json", @@ -1905,7 +1907,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -2285,7 +2287,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc3655aa6818d65bc620d6911f05aa7b6aeb596291e1e9f79e52df85583d1e30" dependencies = [ "rustix 0.38.44", - "windows-targets 0.52.6", + "windows-targets", ] [[package]] @@ -3220,7 +3222,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3279,7 +3281,7 @@ dependencies = [ "portable-atomic", "portable-atomic-util", "serde_core", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -5007,9 +5009,9 @@ dependencies = [ [[package]] name = "papergrid" -version = "0.17.0" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6978128c8b51d8f4080631ceb2302ab51e32cc6e8615f735ee2f83fd269ae3f1" +checksum = "d0984e668274d34691bc2b262ef0d115de5fa9973bcdee7ae32213f93099153e" dependencies = [ "bytecount", "fnv", @@ -5423,17 +5425,6 @@ dependencies = [ "progenitor-macro 0.10.0", ] -[[package]] -name = "progenitor" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2326f73d5326257514712436680ef8da4543ee47c0e9e0d501545c8909ee12e4" -dependencies = [ - "progenitor-client 0.11.2", - "progenitor-impl 0.11.2", - "progenitor-macro 0.11.2", -] - [[package]] name = "progenitor" version = "0.13.0" @@ -5471,21 +5462,6 @@ dependencies = [ "serde_urlencoded", ] -[[package]] -name = "progenitor-client" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71a0beb939758f229cbae70a4889c7c76a4ac0e90f0b1e7ae9b4636a927d1018" -dependencies = [ - "bytes", - "futures-core", - "percent-encoding", - "reqwest 0.12.28", - "serde", - "serde_json", - "serde_urlencoded", -] - [[package]] name = "progenitor-client" version = "0.13.0" @@ -5550,28 +5526,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "progenitor-impl" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f6d9109b04e005bbdec84cacec7e81cc15533f2b5dc505f0defc212d270c15" -dependencies = [ - "heck 0.5.0", - "http", - "indexmap 2.14.0", - "openapiv3", - "proc-macro2", - "quote", - "regex", - "schemars 0.8.22", - "serde", - "serde_json", - "syn 2.0.117", - "thiserror 2.0.18", - "typify 0.4.3", - "unicode-ident", -] - [[package]] name = "progenitor-impl" version = "0.13.0" @@ -5634,24 +5588,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "progenitor-macro" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46596c574831739c661f22923fe587399c61f5e3e79b73cc9a93644c72248d84" -dependencies = [ - "openapiv3", - "proc-macro2", - "progenitor-impl 0.11.2", - "quote", - "schemars 0.8.22", - "serde", - "serde_json", - "serde_tokenstream", - "serde_yaml", - "syn 2.0.117", -] - [[package]] name = "progenitor-macro" version = "0.13.0" @@ -5875,7 +5811,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -6328,7 +6264,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6410,7 +6346,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7301,7 +7237,7 @@ version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" dependencies = [ - "heck 0.5.0", + "heck 0.4.1", "proc-macro2", "quote", "syn 2.0.117", @@ -7532,11 +7468,11 @@ dependencies = [ [[package]] name = "tabled" -version = "0.20.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e39a2ee1fbcd360805a771e1b300f78cc88fec7b8d3e2f71cd37bbf23e725c7d" +checksum = "b5dc662e6da844ad6e428ad16b57967c9d33c82e16bb1c258326c0c078605dff" dependencies = [ - "papergrid 0.17.0", + "papergrid 0.18.0", "tabled_derive 0.11.0", "testing_table", ] @@ -7609,7 +7545,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7618,7 +7554,7 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8c27177b12a6399ffc08b98f76f7c9a1f4fe9fc967c784c5a071fa8d93cf7e1" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7628,7 +7564,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" dependencies = [ "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -8204,7 +8140,7 @@ dependencies = [ [[package]] name = "transceiver-controller" version = "0.1.1" -source = "git+https://github.com/oxidecomputer/transceiver-control?branch=main#81167659157860d6587713b3362b7bc791dfb530" +source = "git+https://github.com/oxidecomputer/transceiver-control?branch=main#e88642c75928f8760ed0f09e7593f849d9bc70ac" dependencies = [ "anyhow", "clap", @@ -8216,7 +8152,7 @@ dependencies = [ "slog", "slog-async", "slog-term", - "tabled 0.20.0", + "tabled 0.21.0", "thiserror 2.0.18", "tokio", "transceiver-decode", @@ -8228,7 +8164,7 @@ dependencies = [ [[package]] name = "transceiver-decode" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/transceiver-control?branch=main#81167659157860d6587713b3362b7bc791dfb530" +source = "git+https://github.com/oxidecomputer/transceiver-control?branch=main#e88642c75928f8760ed0f09e7593f849d9bc70ac" dependencies = [ "schemars 0.8.22", "serde", @@ -8240,7 +8176,7 @@ dependencies = [ [[package]] name = "transceiver-messages" version = "0.1.1" -source = "git+https://github.com/oxidecomputer/transceiver-control?branch=main#81167659157860d6587713b3362b7bc791dfb530" +source = "git+https://github.com/oxidecomputer/transceiver-control?branch=main#e88642c75928f8760ed0f09e7593f849d9bc70ac" dependencies = [ "bitflags 2.11.1", "clap", @@ -9061,7 +8997,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -9146,7 +9082,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.6", + "windows-targets", ] [[package]] @@ -9155,16 +9091,7 @@ version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", + "windows-targets", ] [[package]] @@ -9182,31 +9109,14 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", -] - -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] @@ -9215,96 +9125,48 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - [[package]] name = "winnow" version = "0.5.40" diff --git a/Cargo.toml b/Cargo.toml index 5e42f0087..e92d2d89f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -92,6 +92,7 @@ dropshot-api-manager = "0.7.2" dropshot-api-manager-types = "0.7.2" schemars = { version = "0.8", features = [ "uuid1", "chrono" ] } tokio = { version = "1.52.1", features = ["full"] } +futures = "0.3" serde_repr = "0.1" anyhow = "1.0.102" hyper = "1.9.0" @@ -152,4 +153,4 @@ rev = "3d1fe6ad2df3752dd189ad462c4fdec74ebe25f8" [workspace.dependencies.dpd-client] git = "https://github.com/oxidecomputer/dendrite" -rev = "37992295b5dc708d8f120cee805d67418741b556" +rev = "f27f612dde317f2f05716c761a14500e46581ff7" diff --git a/ddm-api-types/versions/Cargo.toml b/ddm-api-types/versions/Cargo.toml index 22841f295..6cb27f90b 100644 --- a/ddm-api-types/versions/Cargo.toml +++ b/ddm-api-types/versions/Cargo.toml @@ -9,9 +9,7 @@ omicron-common.workspace = true oxnet.workspace = true schemars.workspace = true serde.workspace = true +serde_json.workspace = true serde_repr.workspace = true thiserror.workspace = true uuid.workspace = true - -[dev-dependencies] -serde_json.workspace = true diff --git a/ddm-api-types/versions/src/multicast_support/db.rs b/ddm-api-types/versions/src/multicast_support/db.rs index 3af221ae2..6a8a60027 100644 --- a/ddm-api-types/versions/src/multicast_support/db.rs +++ b/ddm-api-types/versions/src/multicast_support/db.rs @@ -46,9 +46,18 @@ pub struct MulticastRoute { pub path: Vec, } +impl MulticastRoute { + /// Identity used for equality and hashing: which group, from which peer. + /// Excludes `path`, so equality and hashing both key on the same fields + /// and cannot drift as the struct grows. + fn identity(&self) -> (&MulticastOrigin, &Ipv6Addr) { + (&self.origin, &self.nexthop) + } +} + impl PartialEq for MulticastRoute { fn eq(&self, other: &Self) -> bool { - self.origin == other.origin && self.nexthop == other.nexthop + self.identity() == other.identity() } } @@ -56,8 +65,7 @@ impl Eq for MulticastRoute {} impl std::hash::Hash for MulticastRoute { fn hash(&self, state: &mut H) { - self.origin.hash(state); - self.nexthop.hash(state); + self.identity().hash(state); } } diff --git a/ddm-api-types/versions/src/multicast_support/exchange.rs b/ddm-api-types/versions/src/multicast_support/exchange.rs index 20babf392..383b252fe 100644 --- a/ddm-api-types/versions/src/multicast_support/exchange.rs +++ b/ddm-api-types/versions/src/multicast_support/exchange.rs @@ -2,7 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -//! Exchange (wire) types added in version 2 (MULTICAST_SUPPORT). +//! Exchange (wire) types added in version 3 (MULTICAST_SUPPORT). use schemars::JsonSchema; use serde::{Deserialize, Serialize}; diff --git a/ddm-api-types/versions/src/multicast_support/net.rs b/ddm-api-types/versions/src/multicast_support/net.rs index 83fde64cd..a38b71a77 100644 --- a/ddm-api-types/versions/src/multicast_support/net.rs +++ b/ddm-api-types/versions/src/multicast_support/net.rs @@ -3,7 +3,7 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. //! Multicast origin and validated underlay address types added in -//! version 2 (MULTICAST_SUPPORT). +//! version 3 (MULTICAST_SUPPORT). pub use multicast_types::{UnderlayMulticastError, UnderlayMulticastIpv6}; pub use omicron_common::api::external::Vni; @@ -50,26 +50,51 @@ pub struct MulticastOrigin { pub source: Option, } -// Equality and hashing consider only the identity fields (overlay_group, -// underlay_group, vni, source), not metric. This allows metric updates to -// replace existing entries in HashSet-based collections without creating -// duplicates. This type is not used in ordered collections (BTreeSet). -// See #649 for why adding Ord here would require more care. +impl MulticastOrigin { + /// Identity used for equality and hashing: the group, its underlay + /// mapping, VNI, and source. Excludes `metric`, a mutable path-selection + /// attribute, so a metric change updates an existing entry rather than + /// creating a duplicate. Routing both `PartialEq` and `Hash` through this + /// accessor keeps the field set defined once so the two cannot drift. + /// + /// This type is not used in ordered collections (BTreeSet). See #649 for + /// why adding `Ord` here would require more care. + fn identity( + &self, + ) -> (&IpAddr, &UnderlayMulticastIpv6, &Vni, &Option) { + ( + &self.overlay_group, + &self.underlay_group, + &self.vni, + &self.source, + ) + } + + /// Return a stable string key for this origin's identity. + /// + /// Serializes only the identity fields, matching `PartialEq`/`Hash`, so a + /// keyed store overwrites the entry for an origin whose `metric` changed + /// rather than leaving a stale entry under the prior metric. Deriving the + /// key from [`MulticastOrigin::identity`] keeps it from drifting from + /// equality. + /// + /// # Errors + /// + /// Returns an error if the identity fields fail to serialize. + pub fn identity_key(&self) -> Result { + serde_json::to_string(&self.identity()) + } +} + impl PartialEq for MulticastOrigin { fn eq(&self, other: &Self) -> bool { - self.overlay_group == other.overlay_group - && self.underlay_group == other.underlay_group - && self.vni == other.vni - && self.source == other.source + self.identity() == other.identity() } } impl std::hash::Hash for MulticastOrigin { fn hash(&self, state: &mut H) { - self.overlay_group.hash(state); - self.underlay_group.hash(state); - self.vni.hash(state); - self.source.hash(state); + self.identity().hash(state); } } diff --git a/ddm/Cargo.toml b/ddm/Cargo.toml index 510dd9adf..3734e89a7 100644 --- a/ddm/Cargo.toml +++ b/ddm/Cargo.toml @@ -17,6 +17,7 @@ thiserror.workspace = true dropshot.workspace = true schemars.workspace = true tokio.workspace = true +futures.workspace = true anyhow.workspace = true hyper.workspace = true hyper-util.workspace = true @@ -41,7 +42,8 @@ libnet = { workspace = true, optional = true } dpd-client = { workspace = true, optional = true } opte-ioctl = { workspace = true, optional = true } oxide-vpc = { workspace = true, optional = true } +reqwest = { workspace = true, optional = true } [features] default = ["backend"] -backend = ["dep:libnet", "dep:dpd-client", "dep:opte-ioctl", "dep:oxide-vpc"] +backend = ["dep:libnet", "dep:dpd-client", "dep:opte-ioctl", "dep:oxide-vpc", "dep:reqwest"] diff --git a/ddm/src/admin.rs b/ddm/src/admin.rs index 01acee812..c1b75f60c 100644 --- a/ddm/src/admin.rs +++ b/ddm/src/admin.rs @@ -483,30 +483,25 @@ pub(crate) fn do_get_peers( ctx: &Arc>, ) -> HashMap { let ctx = lock!(ctx); - let mut res = HashMap::new(); - for sm in &ctx.peers { - // Compute status first so peer_status() never runs while we hold - // any of the InterfaceState mutexes below. - let status = sm.iface.peer_status(); - let if_index = *lock!(sm.iface.if_index); - let if_name = lock!(sm.iface.if_name).clone(); - let Some(peer) = lock!(sm.iface.peer_identity).clone() else { - continue; - }; - res.insert( - if_index, - PeerInfo { - status, - addr: peer.addr, - host: peer.hostname, - kind: peer.kind, - if_name: if if_name.is_empty() { - None - } else { - Some(if_name) + ctx.peers + .iter() + .filter_map(|sm| { + // Compute status first so peer_status() never runs while we hold + // any of the InterfaceState mutexes below. + let status = sm.iface.peer_status(); + let peer = lock!(sm.iface.peer_identity).clone()?; + let if_index = *lock!(sm.iface.if_index); + let if_name = lock!(sm.iface.if_name).clone(); + Some(( + if_index, + PeerInfo { + status, + addr: peer.addr, + host: peer.hostname, + kind: peer.kind, + if_name: (!if_name.is_empty()).then_some(if_name), }, - }, - ); - } - res + )) + }) + .collect() } diff --git a/ddm/src/db.rs b/ddm/src/db.rs index 77500d69d..2477dcf2b 100644 --- a/ddm/src/db.rs +++ b/ddm/src/db.rs @@ -93,6 +93,24 @@ impl Db { lock!(self.data).imported_mcast.len() } + /// Underlay groups imported via `nexthop`, deduplicated. + /// + /// Filters under the lock and returns only the distinct group addresses, so + /// the caller never clones the full imported set just to keep one peer's + /// routes. This is the non-destructive analog of the next-hop filter in + /// [`Db::remove_nexthop_routes`]. + pub fn mcast_groups_for_nexthop( + &self, + nexthop: Ipv6Addr, + ) -> HashSet { + lock!(self.data) + .imported_mcast + .iter() + .filter(|route| route.nexthop == nexthop) + .map(|route| route.origin.underlay_group.ip()) + .collect() + } + pub fn import(&self, r: &HashSet) { lock!(self.data).imported.extend(r.clone()); } @@ -101,10 +119,6 @@ impl Db { lock!(self.data).imported_tunnel.extend(r.clone()); } - pub fn import_mcast(&self, r: &HashSet) { - lock!(self.data).imported_mcast.extend(r.clone()); - } - pub fn delete_import(&self, r: &HashSet) { let imported = &mut lock!(self.data).imported; for x in r { @@ -119,16 +133,9 @@ impl Db { } } - pub fn delete_import_mcast(&self, r: &HashSet) { - let imported = &mut lock!(self.data).imported_mcast; - for x in r { - imported.remove(x); - } - } - /// Atomically import and delete multicast routes under a single lock, - /// returning the effective difference (additions + removals) against the - /// state before mutation. + /// returning the effective difference as `(additions, removals)` against + /// the state before any mutation. /// /// This avoids a TOCTOU race where concurrent mutations between separate /// lock acquisitions could produce an incorrect view difference. @@ -179,116 +186,86 @@ impl Db { ) -> Result<(), Error> { let tree = self.persistent_data.open_tree(MCAST_ORIGINATE)?; for o in origins { - let entry = serde_json::to_string(o)?; - tree.insert(entry.as_str(), "")?; + // Key by the metric-excluded identity, store the full origin as the + // value. `MulticastOrigin` equality ignores `metric`, so keying by + // identity lets a re-origination with a changed metric overwrite the + // stored entry instead of leaving a stale one under the old metric. + tree.insert( + o.identity_key()?.as_str(), + serde_json::to_string(o)?.as_str(), + )?; } tree.flush()?; Ok(()) } - pub fn originated(&self) -> Result, Error> { - let tree = self.persistent_data.open_tree(ORIGINATE)?; + /// Scan a persistent origin tree, parsing each `(key, value)` pair with + /// `parse` and skipping entries that fail to read or parse. `kind` names the + /// entry kind for log context. + fn scan_origin_tree( + &self, + tree: &str, + kind: &str, + parse: impl Fn(&[u8], &[u8]) -> Result, + ) -> Result, Error> + where + T: Eq + std::hash::Hash, + { + let tree = self.persistent_data.open_tree(tree)?; let result = tree - .scan_prefix(vec![]) + .iter() .filter_map(|item| { - let (key, _value) = match item { + let (key, value) = match item { Ok(item) => item, Err(e) => { error!( self.log, - "db: error ddm originated prefix: {e}" + "db: error fetching ddm {kind} entry: {e}" ); return None; } }; - Some(match Ipv6Net::from_db_key(&key) { - Ok(item) => item, + match parse(key.as_ref(), value.as_ref()) { + Ok(item) => Some(item), Err(e) => { - error!( - self.log, - "db: error parsing ddm origin entry value: {e}" - ); - return None; + error!(self.log, "db: error parsing ddm {kind}: {e}"); + None } - }) + } }) .collect(); Ok(result) } + pub fn originated(&self) -> Result, Error> { + self.scan_origin_tree(ORIGINATE, "origin prefix", |key, _value| { + Ipv6Net::from_db_key(key).map_err(|e| Error::DbKey(e.to_string())) + }) + } + pub fn originated_count(&self) -> Result { Ok(self.originated()?.len()) } pub fn originated_tunnel(&self) -> Result, Error> { - let tree = self.persistent_data.open_tree(TUNNEL_ORIGINATE)?; - let result = tree - .scan_prefix(vec![]) - .filter_map(|item| { - let (key, _value) = match item { - Ok(item) => item, - Err(e) => { - error!( - self.log, - "db: error fetching ddm tunnel origin entry: {e}" - ); - return None; - } - }; - - let value = String::from_utf8_lossy(&key); - let value: TunnelOrigin = match serde_json::from_str(&value) { - Ok(item) => item, - Err(e) => { - error!( - self.log, - "db: error parsing ddm tunnel origin: {e}" - ); - return None; - } - }; - Some(value) - }) - .collect(); - Ok(result) + self.scan_origin_tree(TUNNEL_ORIGINATE, "tunnel origin", |key, _v| { + Ok(serde_json::from_str(&String::from_utf8_lossy(key))?) + }) } pub fn originated_tunnel_count(&self) -> Result { Ok(self.originated_tunnel()?.len()) } + /// Multicast origins originated locally. + /// + /// Each origin is keyed by its metric-excluded identity and stored as the + /// value, so the current metric is read back from the value rather than the + /// key. pub fn originated_mcast(&self) -> Result, Error> { - let tree = self.persistent_data.open_tree(MCAST_ORIGINATE)?; - let result = tree - .scan_prefix(vec![]) - .filter_map(|item| { - let (key, _value) = match item { - Ok(item) => item, - Err(e) => { - error!( - self.log, - "db: error fetching ddm mcast origin entry: {e}" - ); - return None; - } - }; - - let value = String::from_utf8_lossy(&key); - let value: MulticastOrigin = match serde_json::from_str(&value) - { - Ok(item) => item, - Err(e) => { - error!( - self.log, - "db: error parsing ddm mcast origin: {e}" - ); - return None; - } - }; - Some(value) - }) - .collect(); - Ok(result) + self.scan_origin_tree(MCAST_ORIGINATE, "mcast origin", |_key, value| { + Ok(serde_json::from_str(&String::from_utf8_lossy(value))?) + }) } pub fn originated_mcast_count(&self) -> Result { @@ -323,8 +300,9 @@ impl Db { ) -> Result<(), Error> { let tree = self.persistent_data.open_tree(MCAST_ORIGINATE)?; for o in origins { - let entry = serde_json::to_string(o)?; - tree.remove(entry.as_str())?; + // Remove by identity so a withdraw matches regardless of the metric + // the origin was advertised with (e.g. the CLI's default metric). + tree.remove(o.identity_key()?.as_str())?; } tree.flush()?; Ok(()) @@ -333,11 +311,7 @@ impl Db { pub fn remove_nexthop_routes( &self, nexthop: Ipv6Addr, - ) -> ( - HashSet, - HashSet, - HashSet, - ) { + ) -> RemovedNexthopRoutes { let mut data = lock!(self.data); // Routes are generally held in sets to prevent duplication and provide // handy set-algebra operations. @@ -371,7 +345,11 @@ impl Db { data.imported_mcast.remove(x); } - (removed, tnl_removed, mcast_removed) + RemovedNexthopRoutes { + underlay: removed, + tunnel: tnl_removed, + multicast: mcast_removed, + } } pub fn routes_by_vector( @@ -390,6 +368,13 @@ impl Db { } } +/// Routes withdrawn for a next hop, grouped by route family. +pub struct RemovedNexthopRoutes { + pub underlay: HashSet, + pub tunnel: HashSet, + pub multicast: HashSet, +} + #[derive( Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema, )] diff --git a/ddm/src/exchange/mod.rs b/ddm/src/exchange/mod.rs index b00b484b5..ae52de97f 100644 --- a/ddm/src/exchange/mod.rs +++ b/ddm/src/exchange/mod.rs @@ -123,6 +123,7 @@ impl From for Update { Update { underlay: value.underlay, tunnel: value.tunnel, + // V3 protocol doesn't support multicast multicast: None, } } @@ -212,6 +213,7 @@ impl From for PullResponse { PullResponse { underlay: value.underlay, tunnel: value.tunnel, + // V3 protocol doesn't support multicast multicast: None, } } @@ -446,7 +448,7 @@ mod tests { use ddm_api_types::net::{MulticastOrigin, UnderlayMulticastIpv6, Vni}; use std::net::Ipv6Addr; - fn sample_multicast_update() -> MulticastUpdate { + fn multicast_update() -> MulticastUpdate { let origin = MulticastOrigin { overlay_group: "233.252.0.1".parse().unwrap(), underlay_group: UnderlayMulticastIpv6::new( @@ -457,14 +459,14 @@ mod tests { metric: 0, source: None, }; - let pv = MulticastPathVector { + let path_vector = MulticastPathVector { origin, path: vec![MulticastPathHop::new( "router-1".into(), Ipv6Addr::LOCALHOST, )], }; - MulticastUpdate::announce([pv].into_iter().collect()) + MulticastUpdate::announce([path_vector].into_iter().collect()) } #[test] @@ -472,7 +474,7 @@ mod tests { let update = Update { underlay: None, tunnel: None, - multicast: Some(sample_multicast_update()), + multicast: Some(multicast_update()), }; let json = serde_json::to_string(&update).unwrap(); let back: Update = serde_json::from_str(&json).unwrap(); @@ -485,7 +487,7 @@ mod tests { let update = Update { underlay: None, tunnel: None, - multicast: Some(sample_multicast_update()), + multicast: Some(multicast_update()), }; let json = serde_json::to_string(&update).unwrap(); // A V3 peer would deserialize this as UpdateV3, silently @@ -519,14 +521,14 @@ mod tests { metric: 0, source: None, }; - let pv = MulticastPathVector { + let path_vector = MulticastPathVector { origin, path: vec![], }; let resp = PullResponse { underlay: None, tunnel: None, - multicast: Some([pv].into_iter().collect()), + multicast: Some([path_vector].into_iter().collect()), }; let json = serde_json::to_string(&resp).unwrap(); let back: PullResponse = serde_json::from_str(&json).unwrap(); @@ -545,14 +547,14 @@ mod tests { metric: 0, source: None, }; - let pv = MulticastPathVector { + let path_vector = MulticastPathVector { origin, path: vec![], }; let resp = PullResponse { underlay: None, tunnel: None, - multicast: Some([pv].into_iter().collect()), + multicast: Some([path_vector].into_iter().collect()), }; let json = serde_json::to_string(&resp).unwrap(); // V3 peer drops the multicast field. @@ -577,7 +579,7 @@ mod tests { let update = Update { underlay: None, tunnel: None, - multicast: Some(sample_multicast_update()), + multicast: Some(multicast_update()), }; let v3 = UpdateV3::from(update); let back = Update::from(v3); diff --git a/ddm/src/exchange/runtime.rs b/ddm/src/exchange/runtime.rs index ac643aec2..f6566b561 100644 --- a/ddm/src/exchange/runtime.rs +++ b/ddm/src/exchange/runtime.rs @@ -55,9 +55,9 @@ pub struct HandlerContext { } impl Update { - /// Build an `Update` whose underlay/tunnel/multicast halves carry the - /// announcements from `pr`. Used by [`pull`] to project a pull response - /// back into the update event stream. + /// Build an [`Update`] whose underlay/tunnel/multicast halves carry the + /// announcements from the [`PullResponse`]. Used by [`pull`] to project a + /// pull response back into the update event stream. fn announce(pr: PullResponse) -> Self { Self { underlay: pr.underlay.map(UnderlayUpdate::announce), @@ -643,13 +643,43 @@ fn handle_update(update: &Update, ctx: &HandlerContext) { .as_ref() .map(|update| update.with_path_element(ctx.ctx.hostname.clone())); - // Add our hop info to multicast path vectors before redistribution + // Multicast loop prevention is asymmetric with the underlay. The + // underlay relies on sender-side split-horizon, skipping any route + // whose nexthop is the destination peer. Multicast relies on + // receiver-side path-vector RPF, where, on receipt, any announcement + // already carrying our router_id is dropped. RPF is the authoritative + // loop guard and is strictly stronger than split-horizon because it + // catches loops of any length rather than only the immediate echo. + // + // We apply that same RPF filter here before redistributing, dropping + // any path vector that already traversed us. Forwarding such a vector + // is harmless to a peer that already has us in its path (its own RPF + // drops it), but would propagate a looped path to a peer that does + // not, inflating its collection of path vectors. + let hostname = &ctx.ctx.hostname; let multicast = update.multicast.as_ref().map(|update| { - let hop = MulticastPathHop::new( - ctx.ctx.hostname.clone(), - ctx.ctx.config.addr, - ); - update.with_hop(hop) + let hop = + MulticastPathHop::new(hostname.clone(), ctx.ctx.config.addr); + let passes_rpf = |path_vector: &&MulticastPathVector| { + !path_vector + .path + .iter() + .any(|hop| &hop.router_id == hostname) + }; + MulticastUpdate { + announce: update + .announce + .iter() + .filter(passes_rpf) + .map(|path_vector| path_vector.with_hop(hop.clone())) + .collect(), + withdraw: update + .withdraw + .iter() + .filter(passes_rpf) + .map(|path_vector| path_vector.with_hop(hop.clone())) + .collect(), + } }); let push = Arc::new(Update { @@ -824,7 +854,7 @@ fn handle_multicast_update(update: &MulticastUpdate, ctx: &HandlerContext) { dbg!( ctx.log, ctx.ctx.config.if_name, - "dropping multicast announce for {:?} - loop detected \ + "dropping multicast announce for {:?}; loop detected \ (path length {})", path_vector.origin.overlay_group, path_vector.path.len(), @@ -839,22 +869,26 @@ fn handle_multicast_update(update: &MulticastUpdate, ctx: &HandlerContext) { }); } - let mut remove = HashSet::new(); - for path_vector in &update.withdraw { - // Empty path is safe: MulticastRoute's PartialEq/Hash exclude - // the path field, so this matches by (origin, nexthop) only. - remove.insert(MulticastRoute { + // Empty path is safe: MulticastRoute's PartialEq/Hash exclude the path + // field, so this matches by (origin, nexthop) only. + let remove: HashSet = update + .withdraw + .iter() + .map(|path_vector| MulticastRoute { origin: path_vector.origin.clone(), nexthop: ctx.peer, path: Vec::new(), - }); - } + }) + .collect(); // Atomic import + delete + diff under a single lock. - // - // DDM stores learned multicast state, which feeds back into Omicron, as - // the latter owns OPTE M2P programming via sled-agent (the M2P table is - // global to xde). - // Learned state is queryable via the DDM admin API (get_multicast_groups). db.update_imported_mcast(&import, &remove); + + // Notify the multicast sweep of each affected underlay group so it + // reconciles the group's DPD members. Only the sweep writes to DPD; + // this handler records the import and signals. + crate::mcast::notify_affected_groups( + import.iter().chain(remove.iter()), + &ctx.ctx.mcast_notify, + ); } diff --git a/ddm/src/lib.rs b/ddm/src/lib.rs index f278b3255..ee6f27ad8 100644 --- a/ddm/src/lib.rs +++ b/ddm/src/lib.rs @@ -6,6 +6,8 @@ pub mod admin; pub mod db; pub mod discovery; pub mod exchange; +#[cfg(feature = "backend")] +pub mod mcast; pub mod oxstats; pub mod sm; #[cfg(all(feature = "backend", target_os = "illumos"))] @@ -15,7 +17,11 @@ pub const COMPONENT_DDM: &str = "ddm"; pub const MOD_ADMIN: &str = "admin"; pub const MOD_EXCHANGE: &str = "exchange"; -/// Returns `None` if the set is empty, otherwise `Some(s)`. +/// Wrap a set in `Some`, treating an empty set as absence. +/// +/// # Returns +/// +/// `None` if `set` is empty, otherwise `Some(set)`. #[cfg(all(feature = "backend", target_os = "illumos"))] pub(crate) fn non_empty( set: std::collections::HashSet, diff --git a/ddm/src/mcast.rs b/ddm/src/mcast.rs new file mode 100644 index 000000000..51093876d --- /dev/null +++ b/ddm/src/mcast.rs @@ -0,0 +1,1184 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Underlay multicast membership programming. +//! +//! Programs the local switch's underlay multicast replication members in the +//! Dendrite (DPD) data plane from the multicast routes DDM has imported from +//! peers. This is the multicast analog of the unicast import-to-DPD path that +//! [`crate::sys::add_underlay_routes`] performs in-process: `ddmd` owns both +//! inputs locally, the imported multicast set ([`Db::imported_mcast`]) and the +//! peer table, so it programs DPD without any cross-daemon coordination. +//! +//! Membership is reconciled by a single periodic sweep over every active +//! underlay group, like the unicast lower-half's resync and `rdb`'s reaper. The +//! control plane never writes multicast members to DPD itself. +//! +//! When a peer's subscription to a group changes, the exchange update handler +//! and peer expiry send the group's address down a notify channel to wake the +//! sweep early. Peer-link resolution does the same for any import that raced +//! ahead of the link. Absent a trigger, the sweep self-ticks on +//! [`RECONCILE_INTERVAL`], which is also the drift-repair backstop. The address +//! on a trigger is only a wake hint: the sweep always reconciles the full set, +//! so a coalesced or missed trigger costs at most one interval of latency. +//! +//! Each sweep recomputes the full, desired member set for every tracked group, +//! repairs any drift, and programs members it could not previously (because the +//! group did not yet exist or a peer link had not resolved). A group whose +//! imports are withdrawn stays in the sweep until its DPD member list is +//! confirmed empty, then drops out, so a withdrawn group is emptied exactly +//! once and the tracked set stays bounded to active and recently active groups. +//! +//! DPD's only member-write surface is a full-list replace, so every member edit +//! is a read-modify-write. Groups reconcile concurrently within a pass, but the +//! sweep is the sole writer and each group is a distinct DPD object, so +//! concurrent edits cannot clobber one another. +//! +//! Each imported [`MulticastRoute`] names the peer (`nexthop`) that advertised +//! a group subscription and that peer is a replication target on this sled. +//! Every next hop is resolved to its switch `(PortId, LinkId)` through the +//! interface the peer was discovered on, and members are aggregated per +//! underlay group. +//! +//! Aggregation keys solely on the underlay group address and discards the +//! overlay group. This is sound because Omicron maps each overlay group to a +//! distinct underlay group, so the mapping is one to one and the underlay +//! address alone identifies a group's replication set. A route's overlay group +//! is carried for diagnostics, not for keying. +//! +//! Omicron owns each underlay group's create and delete. `ddmd` programs only +//! the member set of groups that already exist and authorizes each write +//! against the group's tag, read back from DPD, so it never changes the tag or +//! deletes the group. +//! +//! See [RFD 488] for the multicast architecture. +//! +//! [RFD 488]: https://rfd.shared.oxide.computer/rfd/0488 + +use crate::db::Db; +use crate::sm::{DpdConfig, SmContext}; +use crate::sys::DDM_DPD_TAG; +use ddm_api_types::db::MulticastRoute; +use dpd_client::types::{ + Direction, LinkId, MulticastGroupMember, MulticastGroupUpdateUnderlayEntry, + MulticastTag, PortId, UnderlayMulticastIpv6, +}; +use dpd_client::{Client, ClientState}; +use futures::TryStreamExt; +use futures::future::join_all; +use mg_common::lock; +use reqwest::StatusCode; +use slog::{Logger, debug, error, warn}; +use std::collections::{HashMap, HashSet}; +use std::net::Ipv6Addr; +use std::sync::Arc; +use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender}; +use std::time::Duration; + +/// Interval between the sweep's periodic membership reconcile passes. +/// +/// A trigger wakes the sweep immediately, so this interval governs only the +/// drift-repair backstop: how quickly drift and any change not delivered as a +/// trigger converge into DPD. It is kept coarse to bound idle DPD churn, since +/// membership changes themselves arrive as triggers. +const RECONCILE_INTERVAL: Duration = Duration::from_secs(10); + +/// Per-request timeout for a single DPD member operation. +/// +/// Bounds one GET or PUT against an unresponsive DPD so a stalled request +/// cannot delay the rest of a sweep pass indefinitely. It is set above the +/// expected DPD member operation latency so that it fires only on a genuine +/// stall, and low enough that a group's sequential fetch-then-write pair +/// (`2 * DPD_REQUEST_TIMEOUT`) stays under [`RECONCILE_INTERVAL`], so a single +/// stalled group cannot extend a pass beyond one backstop interval. This +/// stall-detection threshold is reasoned about independently of the +/// convergence cadence set by [`RECONCILE_INTERVAL`]. A timed-out operation is +/// logged distinctly and retried on the next pass. +const DPD_REQUEST_TIMEOUT: Duration = Duration::from_secs(3); + +/// Run the multicast membership sweep. +/// +/// Loops forever on the calling thread, so callers run it in a dedicated thread. +/// +/// Tracks the set of active underlay groups and reconciles them on each pass. +/// `notify_rx` is a wake hint only: a trigger wakes the sweep early, and absent +/// a trigger it self-ticks on [`RECONCILE_INTERVAL`]. Every pass reconciles the +/// full tracked set, so the group address carried by a trigger is not consulted +/// and a coalesced trigger costs at most one interval of latency. +/// +/// The tracked set is the union of every currently imported group and any group +/// still being drained. `reconcile_group` returns `false` only once a +/// withdrawn group's DPD members are confirmed empty, so a group leaves the set +/// exactly once its drain is complete. A re-import re-adds it on the next pass +/// since the control plane writes to the DB before sending its trigger. +pub fn run( + db: Db, + peers: Vec, + dpd: DpdConfig, + rt: Arc, + notify_rx: Receiver, + log: Logger, +) { + let client_state = ClientState { + tag: DDM_DPD_TAG.into(), + log: log.clone(), + }; + // Build the inner HTTP client explicitly to bound each request at + // DPD_REQUEST_TIMEOUT. The progenitor-generated dpd_client defaults to a + // 15s connect and request timeout, which exceeds RECONCILE_INTERVAL. A + // single stalled GET could outlast the whole backstop interval, and a + // sequential fetch-then-write pair could run three times it. Stepping down + // to DPD_REQUEST_TIMEOUT keeps a stalled group's pair under one interval. + let http = reqwest::ClientBuilder::new() + .connect_timeout(DPD_REQUEST_TIMEOUT) + .timeout(DPD_REQUEST_TIMEOUT) + .build() + .expect("failed to build DPD HTTP client"); + let client = Client::new_with_client( + &format!("http://{}:{}", dpd.host, dpd.port), + http, + client_state, + ); + + // On each pass, the sweep reconciles every imported group plus any group + // still draining withdrawn members, so the set stays bounded to active and + // recently active groups rather than growing without limit. We seed it from + // the underlay groups DPD already has members for. + // + // On a fresh start, the imported set and triggers only reference groups + // with live subscriptions, so a group whose imports were withdrawn while + // `ddmd` was down would never re-enter the sweep and its stale replication + // members would persist. Folding those groups in once lets the first pass + // drain any that no peer still imports, while groups still imported simply + // reconcile as usual. + let mut tracked: HashSet = rt + .block_on(client.member_group_ips(&log)) + .into_iter() + .collect(); + + loop { + // The imported set and resolved peer links are the same for every group + // in a pass, so compute them once here rather than per group. + let imported = db.imported_mcast(); + let peer_links = resolve_peer_links(&peers, &log); + + tracked = rt.block_on(reconcile_pass( + tracked, imported, peer_links, &client, &log, + )); + + // Wait for a trigger or the backstop interval, whichever comes first, + // then drain any burst since the next pass reconciles everything + // regardless. + match notify_rx.recv_timeout(RECONCILE_INTERVAL) { + Ok(_) => while notify_rx.try_recv().is_ok() {}, + Err(RecvTimeoutError::Timeout) => {} + Err(RecvTimeoutError::Disconnected) => { + // Unreachable while `ddmd` runs: `main()` owns the original + // `notify_tx` and parks for the daemon's lifetime, so the + // channel cannot close even if every per-peer sender clone is + // torn down. We stop the sweep rather than spin on a closed + // channel if that invariant ever changes. + error!(log, "multicast notify channel closed, stopping sweep"); + break; + } + } + } +} + +/// Signal the multicast sweep to reconcile each distinct underlay group touched +/// by `routes`. +/// +/// The route iterator may repeat a group many times, one entry per next hop. +/// The groups are deduplicated, so the sweep wakes once per affected group. The +/// import and withdraw paths share this so both wake the sweep the same way. +pub(crate) fn notify_affected_groups<'a>( + routes: impl IntoIterator, + notify: &Sender, +) { + let affected: HashSet = routes + .into_iter() + .map(|route| route.origin.underlay_group.ip()) + .collect(); + notify_groups(affected, notify); +} + +/// Wake the multicast sweep once per group in `groups`. +fn notify_groups(groups: HashSet, notify: &Sender) { + for group in groups { + // Best-effort trigger to wake the multicast sweep. The sweep owns the + // receiver for the daemon's lifetime, so this send does not fail during + // normal operation. + let _ = notify.send(group); + } +} + +/// Wake the multicast sweep for every group `peer` advertised once that peer's +/// link resolves. +/// +/// A multicast import already wakes the sweep, but a route imported before the +/// peer link resolved cannot be programmed yet, so it waits out the backstop +/// interval. Waking the peer's groups on resolution closes that window. The +/// imported set is read, not consumed, so this is the non-destructive analog of +/// the [`Db::remove_nexthop_routes`] removal on peer expiry. +pub(crate) fn notify_peer_groups( + db: &Db, + peer: Ipv6Addr, + notify: &Sender, +) { + notify_groups(db.mcast_groups_for_nexthop(peer), notify); +} + +/// Reconcile every tracked group against DPD once, returning the next tracked +/// set. +/// +/// Folds every currently imported group into `tracked`, reconciles the whole +/// set concurrently, and returns only the groups `reconcile_group` reports as +/// still active. A withdrawn group lingers for exactly one pass to empty its DPD +/// members, then drops out on the following pass. Re-importing a dropped group +/// re-adds it here, since Omicron writes to the DB before triggering the sweep. +/// +/// The per-group futures run concurrently on this task rather than spawned, so a +/// group whose DPD call stalls does not serialize the others behind it. The pass +/// still returns only once its slowest group completes, but each request is +/// bounded by [`DPD_REQUEST_TIMEOUT`], so a stall delays the pass by that bound +/// at most rather than blocking it indefinitely. +async fn reconcile_pass( + mut tracked: HashSet, + imported: HashSet, + peer_links: HashMap, + client: &C, + log: &Logger, +) -> HashSet { + for route in imported.iter() { + tracked.insert(route.origin.underlay_group.ip()); + } + + // Borrow once so every per-group future shares the same imports and + // resolved links by reference. + let imported = &imported; + let peer_links = &peer_links; + let reconciled = join_all(tracked.into_iter().map(|group_ip| async move { + let keep = + reconcile_group(group_ip, imported, peer_links, client, log).await; + (group_ip, keep) + })) + .await; + + reconciled + .into_iter() + .filter_map(|(group_ip, keep)| keep.then_some(group_ip)) + .collect() +} + +/// Whether a DPD client error is a request timeout. +/// +/// A timeout surfaces as a transport-level error with no HTTP status, so it is +/// distinguished by inspecting the underlying `reqwest::Error` rather than by +/// status code. +fn is_timeout(e: &dpd_client::Error) -> bool { + matches!(e, dpd_client::Error::CommunicationError(re) if re.is_timeout()) +} + +/// Outcome of writing a group's member list to DPD. +#[derive(Clone)] +enum WriteOutcome { + /// Members were written. + Updated, + /// DPD no longer authorizes the write against the group's tag. + /// + /// The tag, owned by DPD, changed from the value read this pass. On an + /// active group a later pass reads the current tag and retries. On a + /// withdrawn group the group was reassigned, so `ddmd` abandons it rather + /// than retrying. + TagReassigned, + /// The group is absent from DPD. + Gone, + /// The write stalled past [`DPD_REQUEST_TIMEOUT`]. + /// + /// Distinguished from [`WriteOutcome::Failed`] so a genuine stall is + /// surfaced separately, though both retry the group on the next pass. + TimedOut, + /// The write failed for an unexpected, non-timeout reason. + Failed, +} + +/// Outcome of reading a group's state from DPD. +#[derive(Clone)] +enum FetchOutcome { + /// The group exists and its authorization tag and current members were read. + Found(String, Vec), + /// The group does not exist in DPD, either because Omicron has not created + /// it yet or because it has been deleted. + Absent, + /// The read stalled past [`DPD_REQUEST_TIMEOUT`]. + /// + /// Distinguished from [`FetchOutcome::ReadFailed`] so a genuine stall is + /// surfaced separately, though both keep the group tracked for retry. + TimedOut, + /// The read failed transiently for a non-timeout reason, so the group's + /// state is unknown this pass. + ReadFailed, +} + +/// DPD group operations the reconcile loop depends on. +/// +/// Abstracted behind a trait so [`reconcile_group`]'s keep/drop logic can be +/// exercised against a mock, without a live DPD endpoint. The production +/// implementation is [`Client`]. +trait GroupClient { + /// Read an underlay group's current members and authorization tag. + async fn fetch_group( + &self, + log: &Logger, + group_ip: Ipv6Addr, + ) -> FetchOutcome; + + /// Write `members` to an underlay group, authorized by its current `tag`. + async fn write_members( + &self, + log: &Logger, + group_ip: Ipv6Addr, + tag: &str, + members: Vec, + ) -> WriteOutcome; + + /// Underlay groups that currently have members programmed in DPD. + /// + /// Read once at startup to seed the sweep's tracked set. `ddmd` is the sole + /// writer of underlay members on this switch, so every group returned was + /// programmed by `ddmd` (or a prior incarnation) and is safe to fold-in. A + /// failure returns an empty set. Orphan recovery then waits for the next + /// `ddmd` restart whose listing succeeds, since the periodic sweep + /// reconciles only tracked and imported groups and never re-lists DPD. + async fn member_group_ips(&self, log: &Logger) -> Vec; +} + +impl GroupClient for Client { + /// Distinguishes a group that is genuinely absent (`FetchOutcome::Absent`) + /// from one whose state could not be read (`FetchOutcome::ReadFailed`), so a + /// withdrawn group is not dropped from the sweep on a transient read failure + /// before its members are confirmed drained. + async fn fetch_group( + &self, + log: &Logger, + group_ip: Ipv6Addr, + ) -> FetchOutcome { + let underlay_ip = UnderlayMulticastIpv6::from(group_ip); + match self.multicast_group_get_underlay(&underlay_ip).await { + Ok(resp) => { + let resp = resp.into_inner(); + FetchOutcome::Found(resp.tag, resp.members) + } + // The underlay group's create and delete are owned by Omicron, which + // creates the group before traffic flows. Until the group exists + // there are no members to program, so skip it. + Err(e) if e.status() == Some(StatusCode::NOT_FOUND) => { + debug!( + log, + "underlay group {group_ip} does not exist yet, skipping \ + until Omicron creates it" + ); + FetchOutcome::Absent + } + // Surface a stalled read distinctly from other failures. The sweep + // retries the group on its next pass regardless. + Err(e) if is_timeout(&e) => { + warn!( + log, + "get of underlay group {group_ip} timed out after \ + {DPD_REQUEST_TIMEOUT:?}, retrying next pass" + ); + FetchOutcome::TimedOut + } + Err(e) => { + error!(log, "failed to get underlay group {group_ip}: {e}"); + FetchOutcome::ReadFailed + } + } + } + + /// The expected races, a tag change (403) or a deleted group (404), are + /// returned as outcomes rather than logged, leaving the reaction to the + /// caller. + async fn write_members( + &self, + log: &Logger, + group_ip: Ipv6Addr, + tag: &str, + members: Vec, + ) -> WriteOutcome { + let underlay_ip = UnderlayMulticastIpv6::from(group_ip); + let tag = match MulticastTag::try_from(tag.to_string()) { + Ok(tag) => tag, + Err(e) => { + error!( + log, + "tag for underlay group {group_ip} is invalid, skipping \ + update: {e}" + ); + return WriteOutcome::Failed; + } + }; + let body = MulticastGroupUpdateUnderlayEntry { members }; + match self + .multicast_group_update_underlay(&underlay_ip, &tag, &body) + .await + { + Ok(_) => WriteOutcome::Updated, + Err(e) if e.status() == Some(StatusCode::FORBIDDEN) => { + WriteOutcome::TagReassigned + } + Err(e) if e.status() == Some(StatusCode::NOT_FOUND) => { + WriteOutcome::Gone + } + // Surface a stalled write distinctly from other failures. Treated as + // `WriteOutcome::Failed` so the sweep retries it on its next pass. + Err(e) if is_timeout(&e) => { + warn!( + log, + "update of underlay group {group_ip} members timed out \ + after {DPD_REQUEST_TIMEOUT:?}, retrying next pass" + ); + WriteOutcome::TimedOut + } + Err(e) => { + error!( + log, + "failed to update underlay group {group_ip} members: {e}" + ); + WriteOutcome::Failed + } + } + } + + async fn member_group_ips(&self, log: &Logger) -> Vec { + let groups: Vec = match self + .multicast_groups_list_stream(None) + .try_collect() + .await + { + Ok(groups) => groups, + Err(e) => { + warn!( + log, + "could not list multicast groups to seed sweep, relying \ + on imports and the periodic backstop: {e}" + ); + return Vec::new(); + } + }; + groups + .into_iter() + .filter_map(|group| match group { + dpd_client::types::MulticastGroupResponse::Underlay { + group_ip, + members, + .. + } if !members.is_empty() => Some(*group_ip), + _ => None, + }) + .collect() + } +} + +/// Resolve each established peer's underlay address to a switch +/// `(PortId, LinkId)` through the interface it was discovered on. +/// +/// Peers without an established identity, without an interface name, or whose +/// interface does not resolve to a switch link are omitted. +/// +/// A peer omitted here is seen by `group_members` as an unresolved next hop, so +/// a transient resolution failure neither drops a previously programmed member +/// nor blocks a newly resolved one. +fn resolve_peer_links( + peers: &[SmContext], + log: &Logger, +) -> HashMap { + let mut peer_links: HashMap = HashMap::new(); + for sm in peers { + let Some(peer) = lock!(sm.iface.peer_identity).clone() else { + continue; + }; + let if_name = lock!(sm.iface.if_name).clone(); + if if_name.is_empty() { + warn!( + log, + "peer {} has no interface name; omitting as multicast member", + peer.addr + ); + continue; + } + match mg_common::tfport::port_link_from_ifname(&if_name) { + Ok(port_link) => { + peer_links.insert(peer.addr, port_link); + } + Err(e) => warn!( + log, + "cannot resolve peer {} interface {if_name} to a switch link, \ + omitting as multicast member: {e}", + peer.addr + ), + } + } + peer_links +} + +/// Aggregate one underlay group's desired replication members. +/// +/// Returns the member list for `group_ip` and whether any of its next hops +/// failed to resolve this pass, in which case the derived set may be incomplete. +/// +/// Members are derived from each route's `nexthop`. Distinct downstream peers +/// carry distinct next hops, so each becomes its own member. Subscribers reached +/// through the same downstream peer collapse to one member, because a single +/// egress port toward that peer suffices and the next hop handles further +/// fan-out. The path vector is not needed, only the per-node egress set. +fn group_members( + group_ip: Ipv6Addr, + imported: &HashSet, + peer_links: &HashMap, +) -> (Vec, bool) { + let mut members: Vec = Vec::new(); + let mut has_unresolved = false; + for route in imported + .iter() + .filter(|route| route.origin.underlay_group.ip() == group_ip) + { + let Some((port_id, link_id)) = peer_links.get(&route.nexthop) else { + has_unresolved = true; + continue; + }; + // The single (port, link) here is per next hop, not per group: one peer + // is reached over one tfport link. A group fans out to as many links as + // it has distinct downstream peers. + let member = MulticastGroupMember { + port_id: port_id.clone(), + link_id: *link_id, + direction: Direction::Underlay, + }; + if !members.contains(&member) { + members.push(member); + } + } + (members, has_unresolved) +} + +/// Reconcile a single underlay group's members in DPD against the multicast +/// routes DDM has imported, returning whether the group is still active. +/// +/// The group's current members are read fresh from DPD and diffed against the +/// desired set, so the periodic resync repairs member drift. +/// +/// Returns `true` to keep the group tracked, either while it still has imports +/// (as the drift backstop) or whenever its DPD state could not be read this +/// pass, and `false` only once the group has no imports and its DPD member list +/// is confirmed empty, so it drops out of the sweep. +async fn reconcile_group( + group_ip: Ipv6Addr, + imported: &HashSet, + peer_links: &HashMap, + client: &C, + log: &Logger, +) -> bool { + let has_imports = imported + .iter() + .any(|route| route.origin.underlay_group.ip() == group_ip); + let (members, has_unresolved) = + group_members(group_ip, imported, peer_links); + + let (tag, existing) = match client.fetch_group(log, group_ip).await { + FetchOutcome::Found(tag, existing) => (tag, existing), + // The group is absent from DPD, Omicron has not created it or it is + // deleted. There is nothing to program or drain, so keep it tracked + // only while it still has imports, so a later pass programs it once it + // exists. + FetchOutcome::Absent => return has_imports, + // The state is unknown at this pass. Keep the group tracked so the next + // pass retries, whether it is active or still draining withdrawn + // members. A withdrawn group must not drop out here, or stale + // replication would stay programmed until some later re-import tracked + // it again. + FetchOutcome::TimedOut | FetchOutcome::ReadFailed => return true, + }; + + if !has_imports { + // Withdrawn: empty the member list to stop replication, leaving the + // group for Omicron to delete. + if existing.is_empty() { + return false; + } + return match client.write_members(log, group_ip, &tag, Vec::new()).await + { + WriteOutcome::Updated => { + debug!( + log, + "emptied withdrawn underlay group {group_ip} members" + ); + false + } + // Already gone, or recreated under a tag that is no longer what + // we've seen. Either way `ddmd` no longer programs this group. + WriteOutcome::Gone | WriteOutcome::TagReassigned => false, + // Retry the empty on the next pass. + WriteOutcome::TimedOut | WriteOutcome::Failed => true, + }; + } + + // When a next hop did not resolve this pass, the derived set may be missing + // members. Merge it with the group's current members so a transient + // resolution failure neither drops a previously programmed member nor + // blocks adding a newly resolved one. With every next hop resolved, the + // derived set replaces the current members. + let to_write = if has_unresolved { + let merged = union_members(&members, &existing); + if merged.len() > members.len() { + debug!( + log, + "underlay group {group_ip} has unresolved next hops, preserving \ + {} current DPD member(s) beyond the {} derived this pass", + merged.len() - members.len(), + members.len() + ); + } + merged + } else { + members + }; + + if !members_eq(&existing, &to_write) { + match client.write_members(log, group_ip, &tag, to_write).await { + WriteOutcome::Updated => { + debug!(log, "updated underlay group {group_ip} members") + } + WriteOutcome::TagReassigned => warn!( + log, + "tag no longer authorizes underlay group {group_ip}, retrying \ + with a fresh read next pass" + ), + WriteOutcome::Gone + | WriteOutcome::Failed + | WriteOutcome::TimedOut => {} + } + } + + // Active group: keep it tracked so the backstop repairs any later drift. + true +} + +/// Union of two multicast member lists, preserving order and dropping +/// duplicates. Used to merge a derived member set with the group's current DPD +/// members when a next hop did not resolve this pass. +fn union_members( + base: &[MulticastGroupMember], + extra: &[MulticastGroupMember], +) -> Vec { + base.iter() + .chain(extra.iter().filter(|member| !base.contains(member))) + .cloned() + .collect() +} + +/// Compare two multicast member lists for set equality, ignoring order. +fn members_eq(a: &[MulticastGroupMember], b: &[MulticastGroupMember]) -> bool { + a.len() == b.len() && a.iter().all(|member| b.contains(member)) +} + +#[cfg(test)] +mod tests { + use super::*; + use ddm_api_types::net::{MulticastOrigin, UnderlayMulticastIpv6, Vni}; + use std::net::IpAddr; + + fn underlay(last: u16) -> Ipv6Addr { + Ipv6Addr::new(0xff04, 0, 0, 0, 0, 0, 0, last) + } + + fn route(nexthop: Ipv6Addr, group: Ipv6Addr) -> MulticastRoute { + MulticastRoute { + origin: MulticastOrigin { + overlay_group: IpAddr::V6(Ipv6Addr::new( + 0xff0e, 0, 0, 0, 0, 0, 0, 1, + )), + underlay_group: UnderlayMulticastIpv6::new(group).unwrap(), + vni: Vni::DEFAULT_MULTICAST_VNI, + metric: 0, + source: None, + }, + nexthop, + path: Vec::new(), + } + } + + fn rear(port: &str, link: u8) -> (PortId, LinkId) { + ( + PortId::Rear(dpd_client::types::Rear::try_from(port).unwrap()), + LinkId(link), + ) + } + + fn member(port: &str, link: u8) -> MulticastGroupMember { + let (port_id, link_id) = rear(port, link); + MulticastGroupMember { + port_id, + link_id, + direction: Direction::Underlay, + } + } + + #[test] + fn distinct_peers_become_distinct_members() { + let peer_a = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let peer_b = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 2); + let group = underlay(1); + + let imported = + HashSet::from([route(peer_a, group), route(peer_b, group)]); + let peer_links = HashMap::from([ + (peer_a, rear("rear0", 0)), + (peer_b, rear("rear1", 0)), + ]); + + let (members, unresolved) = + group_members(group, &imported, &peer_links); + assert!(!unresolved); + assert_eq!(members.len(), 2); + assert!(members.contains(&member("rear0", 0))); + assert!(members.contains(&member("rear1", 0))); + } + + #[test] + fn distinct_peers_on_same_link_collapse_to_one_member() { + let peer_a = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let peer_b = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 2); + let group = underlay(1); + + // Two distinct peers that resolve to the same switch link. Replicating + // twice out one (PortId, LinkId) would duplicate delivery on that link, + // so the members collapse to one. The dedup keys on the resolved link, + // not on the next hop. + let imported = + HashSet::from([route(peer_a, group), route(peer_b, group)]); + let peer_links = HashMap::from([ + (peer_a, rear("rear0", 0)), + (peer_b, rear("rear0", 0)), + ]); + + let (members, unresolved) = + group_members(group, &imported, &peer_links); + assert!(!unresolved); + assert_eq!(members, vec![member("rear0", 0)]); + } + + #[test] + fn unresolved_nexthop_yields_no_members_and_sets_flag() { + let peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let group = underlay(7); + + let imported = HashSet::from([route(peer, group)]); + // No peer_links entry: next hop is unresolved. + let peer_links = HashMap::new(); + + let (members, unresolved) = + group_members(group, &imported, &peer_links); + assert!(members.is_empty()); + assert!(unresolved); + } + + #[test] + fn mixed_resolution_yields_resolved_members_and_sets_flag() { + let resolved = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let unresolved_peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 2); + let group = underlay(3); + + // One next hop resolves and one does not. The resolved peer contributes a + // member, and the group is still flagged so the reconcile merges with the + // group's current DPD members rather than dropping the unresolved one. + let imported = HashSet::from([ + route(resolved, group), + route(unresolved_peer, group), + ]); + let peer_links = HashMap::from([(resolved, rear("rear0", 0))]); + + let (members, unresolved) = + group_members(group, &imported, &peer_links); + assert!(unresolved); + assert_eq!(members, vec![member("rear0", 0)]); + } + + /// Mock DPD that returns preset fetch and write outcomes, and records every + /// member list written so a test can assert the reconcile's keep/drop + /// decision and whether it wrote at all. + /// + /// A landed write (`WriteOutcome::Updated`) updates the stored fetch state + /// to the members just written, so a later fetch reflects it. This models + /// DPD's read-after-write semantics and lets a multi-pass test observe a + /// group's drain across passes. + struct MockDpd { + fetch: std::sync::Mutex, + write_outcome: WriteOutcome, + writes: std::sync::Mutex>>, + member_groups: Vec, + } + + impl MockDpd { + fn new(fetch: FetchOutcome, write_outcome: WriteOutcome) -> Self { + Self { + fetch: std::sync::Mutex::new(fetch), + write_outcome, + writes: std::sync::Mutex::new(Vec::new()), + member_groups: Vec::new(), + } + } + + fn with_member_groups(mut self, groups: Vec) -> Self { + self.member_groups = groups; + self + } + + fn writes(&self) -> Vec> { + self.writes.lock().unwrap().clone() + } + } + + impl GroupClient for MockDpd { + async fn fetch_group( + &self, + _log: &Logger, + _group_ip: Ipv6Addr, + ) -> FetchOutcome { + self.fetch.lock().unwrap().clone() + } + + async fn write_members( + &self, + _log: &Logger, + _group_ip: Ipv6Addr, + tag: &str, + members: Vec, + ) -> WriteOutcome { + self.writes.lock().unwrap().push(members.clone()); + if matches!(self.write_outcome, WriteOutcome::Updated) { + *self.fetch.lock().unwrap() = + FetchOutcome::Found(tag.to_string(), members); + } + self.write_outcome.clone() + } + + async fn member_group_ips(&self, _log: &Logger) -> Vec { + self.member_groups.clone() + } + } + + fn found(members: Vec) -> FetchOutcome { + FetchOutcome::Found("tag".to_string(), members) + } + + fn reconcile( + group: Ipv6Addr, + imported: &HashSet, + peer_links: &HashMap, + mock: &MockDpd, + ) -> bool { + let log = Logger::root(slog::Discard, slog::o!()); + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + rt.block_on(reconcile_group(group, imported, peer_links, mock, &log)) + } + + fn run_pass( + tracked: HashSet, + imported: &HashSet, + peer_links: &HashMap, + mock: &MockDpd, + ) -> HashSet { + let log = Logger::root(slog::Discard, slog::o!()); + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + rt.block_on(reconcile_pass( + tracked, + imported.clone(), + peer_links.clone(), + mock, + &log, + )) + } + + /// Drives the sweep's cross-pass carry-over invariant: an active group is + /// tracked, a withdraw lingers one pass to empty its members then drops, and + /// a re-import re-adds and reprograms it. This exercises the tracked-set + /// state machine that the run loop builds on. + #[test] + fn pass_drains_withdrawn_group_then_drops_and_readds_on_reimport() { + let peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let group = underlay(1); + let active = HashSet::from([route(peer, group)]); + let withdrawn = HashSet::new(); + let peer_links = HashMap::from([(peer, rear("rear0", 0))]); + + // DPD already holds the derived member, so the group starts active and + // in sync with the imported set. + let mock = MockDpd::new( + found(vec![member("rear0", 0)]), + WriteOutcome::Updated, + ); + + // Pass 1: imported and already in sync, so the group is tracked (no + // write occurrs). + let tracked = run_pass(HashSet::new(), &active, &peer_links, &mock); + assert_eq!(tracked, HashSet::from([group])); + assert!(mock.writes().is_empty()); + + // Pass 2: withdrawn ~ the group carries over from pass 1, its members + // are emptied, and then it drops out of the tracked set. + let tracked = run_pass(tracked, &withdrawn, &peer_links, &mock); + assert!(tracked.is_empty()); + assert_eq!(mock.writes(), vec![Vec::::new()]); + + // Pass 3: re-imported ~ the dropped group is re-added and reprogrammed, + // since DPD now holds no members for it. + let tracked = run_pass(tracked, &active, &peer_links, &mock); + assert_eq!(tracked, HashSet::from([group])); + assert_eq!(mock.writes(), vec![Vec::new(), vec![member("rear0", 0)]]); + } + + /// A group whose imports were withdrawn while `ddmd` was down has no entry + /// in the imported set or any trigger, so only the startup seed can + /// re-initialize it. + #[test] + fn startup_seeds_tracked_from_dpd_and_drains_orphans() { + let group = underlay(9); + let mock = MockDpd::new( + found(vec![member("rear0", 0)]), + WriteOutcome::Updated, + ) + .with_member_groups(vec![group]); + let log = Logger::root(slog::Discard, slog::o!()); + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + let seeded: HashSet = rt + .block_on(mock.member_group_ips(&log)) + .into_iter() + .collect(); + assert!(seeded.contains(&group)); + + let next = run_pass(seeded, &HashSet::new(), &HashMap::new(), &mock); + assert_eq!(mock.writes(), vec![Vec::::new()]); + assert!(next.is_empty()); + } + + #[test] + fn absent_group_with_imports_stays_tracked() { + let peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let group = underlay(1); + let imported = HashSet::from([route(peer, group)]); + let peer_links = HashMap::from([(peer, rear("rear0", 0))]); + let mock = MockDpd::new(FetchOutcome::Absent, WriteOutcome::Updated); + + // Omicron has not created the group yet, so there is nothing to program, + // but it stays tracked so a later pass programs it once it exists. + assert!(reconcile(group, &imported, &peer_links, &mock)); + assert!(mock.writes().is_empty()); + } + + #[test] + fn absent_group_without_imports_drops() { + let group = underlay(1); + let imported = HashSet::new(); + let peer_links = HashMap::new(); + let mock = MockDpd::new(FetchOutcome::Absent, WriteOutcome::Updated); + + assert!(!reconcile(group, &imported, &peer_links, &mock)); + assert!(mock.writes().is_empty()); + } + + #[test] + fn withdrawn_group_with_read_failure_stays_tracked() { + let group = underlay(1); + let imported = HashSet::new(); + let peer_links = HashMap::new(); + let mock = + MockDpd::new(FetchOutcome::ReadFailed, WriteOutcome::Updated); + + // A withdrawn group must not drop out on a transient read failure, or + // its stale replication would stay programmed until a later re-import. + assert!(reconcile(group, &imported, &peer_links, &mock)); + assert!(mock.writes().is_empty()); + } + + #[test] + fn withdrawn_group_with_read_timeout_stays_tracked() { + let group = underlay(1); + let imported = HashSet::new(); + let peer_links = HashMap::new(); + let mock = MockDpd::new(FetchOutcome::TimedOut, WriteOutcome::Updated); + + // A read stall is treated like any other transient read failure: the + // withdrawn group stays tracked so a later pass can drain it. + assert!(reconcile(group, &imported, &peer_links, &mock)); + assert!(mock.writes().is_empty()); + } + + #[test] + fn withdrawn_group_with_no_members_drops_without_writing() { + let group = underlay(1); + let imported = HashSet::new(); + let peer_links = HashMap::new(); + let mock = MockDpd::new(found(Vec::new()), WriteOutcome::Updated); + + assert!(!reconcile(group, &imported, &peer_links, &mock)); + assert!(mock.writes().is_empty()); + } + + #[test] + fn withdrawn_group_with_members_is_emptied_then_drops() { + let group = underlay(1); + let imported = HashSet::new(); + let peer_links = HashMap::new(); + let mock = MockDpd::new( + found(vec![member("rear0", 0)]), + WriteOutcome::Updated, + ); + + assert!(!reconcile(group, &imported, &peer_links, &mock)); + assert_eq!(mock.writes(), vec![Vec::::new()]); + } + + #[test] + fn withdrawn_group_with_empty_write_failure_stays_tracked() { + let group = underlay(1); + let imported = HashSet::new(); + let peer_links = HashMap::new(); + let mock = + MockDpd::new(found(vec![member("rear0", 0)]), WriteOutcome::Failed); + + assert!(reconcile(group, &imported, &peer_links, &mock)); + assert_eq!(mock.writes(), vec![Vec::::new()]); + } + + #[test] + fn withdrawn_group_with_empty_write_timeout_stays_tracked() { + let group = underlay(1); + let imported = HashSet::new(); + let peer_links = HashMap::new(); + let mock = MockDpd::new( + found(vec![member("rear0", 0)]), + WriteOutcome::TimedOut, + ); + + // The empty write stalled, so the group stays tracked to retry the + // drain on the next pass. + assert!(reconcile(group, &imported, &peer_links, &mock)); + assert_eq!(mock.writes(), vec![Vec::::new()]); + } + + #[test] + fn withdrawn_group_with_tag_reassigned_on_empty_drops() { + let group = underlay(1); + let imported = HashSet::new(); + let peer_links = HashMap::new(); + let mock = MockDpd::new( + found(vec![member("rear0", 0)]), + WriteOutcome::TagReassigned, + ); + + // The group was reassigned under a tag we no longer hold, so `ddmd` + // abandons it rather than retrying. + // + // This is distinct from the active-group case, where a reassigned tag + // stays tracked for a fresh read. + assert!(!reconcile(group, &imported, &peer_links, &mock)); + assert_eq!(mock.writes(), vec![Vec::::new()]); + } + + #[test] + fn active_group_with_matching_members_skips_write() { + let peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let group = underlay(1); + let imported = HashSet::from([route(peer, group)]); + let peer_links = HashMap::from([(peer, rear("rear0", 0))]); + let mock = MockDpd::new( + found(vec![member("rear0", 0)]), + WriteOutcome::Updated, + ); + + assert!(reconcile(group, &imported, &peer_links, &mock)); + assert!(mock.writes().is_empty()); + } + + #[test] + fn active_group_with_drifted_members_writes_derived_set() { + let peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let group = underlay(1); + let imported = HashSet::from([route(peer, group)]); + let peer_links = HashMap::from([(peer, rear("rear0", 0))]); + let mock = MockDpd::new(found(Vec::new()), WriteOutcome::Updated); + + assert!(reconcile(group, &imported, &peer_links, &mock)); + assert_eq!(mock.writes(), vec![vec![member("rear0", 0)]]); + } + + #[test] + fn active_group_with_unresolved_nexthop_preserves_existing_member() { + let peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let group = underlay(1); + let imported = HashSet::from([route(peer, group)]); + // No peer_links entry: the next hop is unresolved this pass. + let peer_links = HashMap::new(); + let mock = MockDpd::new( + found(vec![member("rear0", 0)]), + WriteOutcome::Updated, + ); + + // The derived set is empty because the next hop did not resolve, but + // the merge with current DPD members keeps the programmed member, so no + // destructive write occurs. + assert!(reconcile(group, &imported, &peer_links, &mock)); + assert!(mock.writes().is_empty()); + } + + #[test] + fn active_group_with_tag_reassigned_stays_tracked() { + let peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let group = underlay(1); + let imported = HashSet::from([route(peer, group)]); + let peer_links = HashMap::from([(peer, rear("rear0", 0))]); + let mock = MockDpd::new(found(Vec::new()), WriteOutcome::TagReassigned); + + // The write was rejected because the tag changed, but the group is + // still active, so it stays tracked to retry with a fresh read next + // pass. + assert!(reconcile(group, &imported, &peer_links, &mock)); + assert_eq!(mock.writes(), vec![vec![member("rear0", 0)]]); + } + + #[test] + fn notify_collapses_routes_to_one_trigger_per_group() { + let group_a = underlay(1); + let group_b = underlay(2); + let peer_a = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let peer_b = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 2); + + // Two next hops on group_a and one on group_b. The sweep should wake + // once per distinct group, not once per route. + let routes = [ + route(peer_a, group_a), + route(peer_b, group_a), + route(peer_a, group_b), + ]; + + let (tx, rx) = std::sync::mpsc::channel(); + notify_affected_groups(routes.iter(), &tx); + drop(tx); + + let signalled: Vec = rx.into_iter().collect(); + assert_eq!(signalled.len(), 2); + assert_eq!( + signalled.into_iter().collect::>(), + HashSet::from([group_a, group_b]) + ); + } +} diff --git a/ddm/src/sm/mod.rs b/ddm/src/sm/mod.rs index 1da896f65..c1a5a1a57 100644 --- a/ddm/src/sm/mod.rs +++ b/ddm/src/sm/mod.rs @@ -253,6 +253,11 @@ pub struct SmContext { pub hostname: String, pub iface: Arc, pub stats: Arc, + /// Notifies the [`crate::mcast`] sweep that an underlay group's imported + /// membership changed, by sending the group's address. The sweep wakes early + /// to reconcile the group's DPD members, so the control plane never touches + /// DPD directly. + pub mcast_notify: Sender, pub log: Logger, } diff --git a/ddm/src/sm/state.rs b/ddm/src/sm/state.rs index 4a48a7b13..9ea0e55b1 100644 --- a/ddm/src/sm/state.rs +++ b/ddm/src/sm/state.rs @@ -170,6 +170,14 @@ impl State for Solicit { self.ctx.config.if_name, "transition solicit -> exchange" ); + // The peer is now established on this link, so wake the + // multicast sweep for any of its groups whose import raced + // ahead of resolution. + crate::mcast::notify_peer_groups( + &self.ctx.db, + addr, + &self.ctx.mcast_notify, + ); return ( Box::new(Exchange::new( self.ctx.clone(), @@ -299,8 +307,11 @@ impl Exchange { ) { exchange_thread.abort(); self.ctx.iface.clear_peer(); - let (to_remove, to_remove_tnl, to_remove_mcast) = - self.ctx.db.remove_nexthop_routes(self.peer); + let crate::db::RemovedNexthopRoutes { + underlay: to_remove, + tunnel: to_remove_tnl, + multicast: to_remove_mcast, + } = self.ctx.db.remove_nexthop_routes(self.peer); let mut routes: Vec = Vec::new(); for x in &to_remove { let mut r: crate::sys::Route = x.clone().into(); @@ -326,6 +337,15 @@ impl Exchange { to_remove_tnl ); } + + // The expired peer is gone from the imported set, so notify the + // multicast sweep of each affected underlay group. The sweep drops the + // peer's replication membership from DPD. + crate::mcast::notify_affected_groups( + to_remove_mcast.iter(), + &self.ctx.mcast_notify, + ); + // if we're a transit router propagate withdraws for the // expired peer. if self.ctx.config.kind == RouterKind::Transit { @@ -594,13 +614,13 @@ impl State for Exchange { Event::Admin(AdminEvent::Announce(PrefixSet::Multicast( groups, ))) => { - // Convert `MulticastOrigin` to `MulticastPathVector` with - // our hop info + // Build a `MulticastPathVector` for each origin, recording + // our hop in the path. let hop = MulticastPathHop::new( self.ctx.hostname.clone(), self.ctx.config.addr, ); - let pvs: HashSet<_> = groups + let path_vectors: HashSet<_> = groups .iter() .map(|origin| { ddm_api_types::exchange::MulticastPathVector { @@ -613,7 +633,7 @@ impl State for Exchange { if let Err(e) = crate::exchange::announce_multicast( &self.ctx, self.ctx.config.clone(), - pvs, + path_vectors, self.peer, self.version, self.ctx.rt.clone(), @@ -622,8 +642,7 @@ impl State for Exchange { err!( self.log, self.ctx.config.if_name, - "announce multicast: {}", - e, + "announce multicast: {e}", ); wrn!( self.log, @@ -644,12 +663,13 @@ impl State for Exchange { Event::Admin(AdminEvent::Withdraw(PrefixSet::Multicast( groups, ))) => { - // Convert MulticastOrigin to MulticastPathVector for withdrawal + // Build a `MulticastPathVector` for each origin, recording + // our hop in the path. let hop = MulticastPathHop::new( self.ctx.hostname.clone(), self.ctx.config.addr, ); - let pvs: HashSet<_> = groups + let path_vectors: HashSet<_> = groups .iter() .map(|origin| { ddm_api_types::exchange::MulticastPathVector { @@ -662,7 +682,7 @@ impl State for Exchange { if let Err(e) = crate::exchange::withdraw_multicast( &self.ctx, self.ctx.config.clone(), - pvs, + path_vectors, self.peer, self.version, self.ctx.rt.clone(), @@ -766,6 +786,7 @@ impl State for Exchange { event, ); } + if !push.withdraw.is_empty() && let Err(e) = crate::exchange::withdraw_underlay( &self.ctx, @@ -798,7 +819,7 @@ impl State for Exchange { ); } } - // Handle multicast redistribution + if let Some(push) = update.multicast { if !push.announce.is_empty() && let Err(e) = crate::exchange::announce_multicast( @@ -831,6 +852,7 @@ impl State for Exchange { event, ); } + if !push.withdraw.is_empty() && let Err(e) = crate::exchange::withdraw_multicast( &self.ctx, @@ -896,6 +918,15 @@ impl State for Exchange { Event::Neighbor(NeighborEvent::Advertise((addr, version))) => { self.peer = addr; self.version = version; + // Re-advertisement may carry a new address, so wake the + // multicast sweep for the peer's groups under it. Routes + // still keyed on the prior address are not withdrawn here. + // Peer expiry and the periodic sweep reconcile those. + crate::mcast::notify_peer_groups( + &self.ctx.db, + addr, + &self.ctx.mcast_notify, + ); } } } diff --git a/ddm/src/sys.rs b/ddm/src/sys.rs index d300f8ada..ee9cdee49 100644 --- a/ddm/src/sys.rs +++ b/ddm/src/sys.rs @@ -24,7 +24,9 @@ use ::{ std::collections::HashMap, }; -const DDM_DPD_TAG: &str = "ddmd"; +/// Client identity tag carried in `dpd_client::ClientState` to identify +/// `ddmd` to DPD, distinct from any per-group authorization tag DPD owns. +pub(crate) const DDM_DPD_TAG: &str = "ddmd"; #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] pub struct Route { @@ -188,7 +190,7 @@ pub fn add_routes_dendrite( // TODO this assumes ddm only operates on rear ports, which will not be // true for multi-rack deployments. if tfport.kind != TfportKind::Rear { - err!(log, ifname, "expected tfportrear"); + err!(log, ifname, "expected rear tfport, got {:?}", tfport.kind); continue; } diff --git a/ddmd/src/main.rs b/ddmd/src/main.rs index 87c349aa3..f2a5f9078 100644 --- a/ddmd/src/main.rs +++ b/ddmd/src/main.rs @@ -151,14 +151,29 @@ async fn run() { .to_string_lossy() .to_string(); + // Notify channel into the multicast membership sweep. Each state machine's + // context holds the sender and signals a group's address when its imported + // membership changes. The sweep started by start_mcast_sweep owns the + // receiver and wakes early to reconcile the full tracked set. + let (notify_tx, notify_rx) = std::sync::mpsc::channel::(); + let (sms, event_channels) = - start_state_machines(&arg, &db, &dpd, &hostname, &rt, &log); + start_state_machines(&arg, &db, &dpd, &hostname, &rt, ¬ify_tx, &log); termination_handler(db.clone(), dpd.clone(), rt.clone(), log.clone()); let router_stats = Arc::new(RouterStats::default()); let peers: Vec = sms.iter().map(|x| x.ctx.clone()).collect(); + start_mcast_sweep( + notify_rx, + dpd.clone(), + db.clone(), + peers.clone(), + rt.clone(), + log.clone(), + ); + let stats_handler = if arg.with_stats { if let (Some(rack_uuid), Some(sled_uuid)) = (arg.rack_uuid, arg.sled_uuid) @@ -218,6 +233,7 @@ fn start_state_machines( dpd: &Option, hostname: &str, rt: &Arc, + notify_tx: &std::sync::mpsc::Sender, log: &Logger, ) -> ( Vec, @@ -258,6 +274,7 @@ fn start_state_machines( rt: rt.clone(), iface: Arc::new(InterfaceState::default()), stats: Arc::new(ddm::sm::SessionStats::default()), + mcast_notify: notify_tx.clone(), }; let sm = StateMachine { ctx, rx: Some(rx) }; @@ -294,6 +311,7 @@ fn start_state_machines( _dpd: &Option, _hostname: &str, _rt: &Arc, + _notify_tx: &std::sync::mpsc::Sender, _log: &Logger, ) -> ( Vec, @@ -302,7 +320,50 @@ fn start_state_machines( (Vec::new(), Vec::new()) } -/// Install a Ctrl-C handler that withdraws ddmd's imported routes from the +/// Spawn the underlay multicast membership sweep, the multicast analog of the +/// unicast import-to-DPD path `ddmd` already performs in-process. The sweep +/// runs on a dedicated thread, reconciling every tracked group on each pass, +/// woken early by a trigger and otherwise self-ticking on a periodic backstop. +/// +/// Takes `notify_rx` by value so any path that does not start the sweep drops +/// it, closing the channel and making the state machines' notify sends fail +/// fast rather than accumulate as unread. +#[cfg(all(feature = "backend", target_os = "illumos"))] +fn start_mcast_sweep( + notify_rx: std::sync::mpsc::Receiver, + dpd: Option, + db: Db, + peers: Vec, + rt: Arc, + log: Logger, +) { + let Some(dpd) = dpd else { + // No backend: returning drops notify_rx and closes the channel. + return; + }; + let task_log = log.clone(); + if let Err(e) = std::thread::Builder::new() + .name("ddm-mcast-members".into()) + .spawn(move || ddm::mcast::run(db, peers, dpd, rt, notify_rx, task_log)) + { + error!(log, "failed to spawn multicast membership sweep: {e}"); + } +} + +/// Non-illumos variant: underlay multicast replicates only on a switch, so the +/// sweep never starts. Consuming `notify_rx` drops it, closing the channel. +#[cfg(not(all(feature = "backend", target_os = "illumos")))] +fn start_mcast_sweep( + _notify_rx: std::sync::mpsc::Receiver, + _dpd: Option, + _db: Db, + _peers: Vec, + _rt: Arc, + _log: Logger, +) { +} + +/// Install a Ctrl-C handler that withdraws `ddmd`'s imported routes from the /// kernel before exiting. On non-illumos builds there are no kernel routes /// to withdraw, so the handler just exits cleanly. fn termination_handler( diff --git a/mg-common/src/net.rs b/mg-common/src/net.rs index 7b551e0fd..bda44de9f 100644 --- a/mg-common/src/net.rs +++ b/mg-common/src/net.rs @@ -26,7 +26,7 @@ impl From for TunnelOrigin { fn from(value: TunnelOriginV2) -> Self { // TunnelOriginV2 is the DDMv2 wire shape, frozen by protocol // contract. If this destructure stops compiling, the V2 - // contract has been violated upstream; there is no + // contract has been violated upstream — there is no // #[serde(skip)] escape valve for a wire-format type. let TunnelOriginV2 { overlay_prefix, diff --git a/mg-common/src/tfport.rs b/mg-common/src/tfport.rs index 62379de72..b758b181f 100644 --- a/mg-common/src/tfport.rs +++ b/mg-common/src/tfport.rs @@ -74,8 +74,8 @@ pub struct TfportName { /// /// # Errors /// -/// Returns a human-readable message if `name` lacks the `tfport` prefix, has -/// an unrecognized device kind, or has malformed port/link/vlan fields. +/// Returns an error if `name` lacks the `tfport` prefix, has an unrecognized +/// device kind, or has malformed port/link/vlan fields. pub fn parse_tfport_name(name: &str) -> Result { let body = name.strip_prefix(TFPORT_DEVICE_PREFIX).ok_or_else(|| { format!("{name} missing expected prefix {TFPORT_DEVICE_PREFIX}") @@ -130,8 +130,8 @@ pub fn parse_tfport_name(name: &str) -> Result { /// /// # Errors /// -/// Returns a human-readable message if the synthesized port name is not a valid -/// dpd `qsfp` or `rear` port identifier. +/// Returns an error if the synthesized port name is not a valid dpd `qsfp` or +/// `rear` port identifier. /// /// [`PortId`]: dpd_client::types::PortId pub fn tfport_port_id( @@ -140,7 +140,7 @@ pub fn tfport_port_id( ) -> Result { use dpd_client::types; - let port_name = format!("{}{}", kind.token(), port); + let port_name = format!("{}{port}", kind.token()); match kind { TfportKind::Qsfp => types::Qsfp::try_from(&port_name) .map(types::PortId::Qsfp) @@ -151,14 +151,35 @@ pub fn tfport_port_id( } } +/// Resolve a tfport datalink name (e.g. `tfportrear0_0`) to the dpd +/// `(PortId, LinkId)` pair that names the switch port and link. +/// +/// # Errors +/// +/// Returns an error if `ifname` is not a valid tfport datalink name or its +/// kind and port do not form a valid dpd port identifier. +/// +/// [`PortId`]: dpd_client::types::PortId +/// [`LinkId`]: dpd_client::types::LinkId +pub fn port_link_from_ifname( + ifname: &str, +) -> Result<(dpd_client::types::PortId, dpd_client::types::LinkId), String> { + let tfport = parse_tfport_name(ifname)?; + let port_id = tfport_port_id(tfport.kind, tfport.port)?; + // Breakout lanes surface as distinct datalinks (`qsfp0_0`, `qsfp0_1`, ...), + // sharing one `PortId`. The parsed link is the lane, which is the dpd + // `LinkId`. + let link_id = dpd_client::types::LinkId(tfport.link); + Ok((port_id, link_id)) +} + #[cfg(test)] mod tests { - use super::TfportKind::{Qsfp, Rear}; - use super::{TfportName, parse_tfport_name}; + use super::*; use proptest::prelude::*; fn name( - kind: super::TfportKind, + kind: TfportKind, port: u8, link: u8, vlan: Option, @@ -176,25 +197,25 @@ mod tests { // Valid qsfp (front-panel) names. assert_eq!( parse_tfport_name("tfportqsfp10_0").unwrap(), - name(Qsfp, 10, 0, None) + name(TfportKind::Qsfp, 10, 0, None) ); assert_eq!( parse_tfport_name("tfportqsfp10_0.100").unwrap(), - name(Qsfp, 10, 0, Some(100)) + name(TfportKind::Qsfp, 10, 0, Some(100)) ); assert_eq!( parse_tfport_name("tfportqsfp1_1").unwrap(), - name(Qsfp, 1, 1, None) + name(TfportKind::Qsfp, 1, 1, None) ); // Valid rear (backplane) names. assert_eq!( parse_tfport_name("tfportrear0_0").unwrap(), - name(Rear, 0, 0, None) + name(TfportKind::Rear, 0, 0, None) ); assert_eq!( parse_tfport_name("tfportrear31_0.200").unwrap(), - name(Rear, 31, 0, Some(200)) + name(TfportKind::Rear, 31, 0, Some(200)) ); // Malformed names. @@ -217,9 +238,10 @@ mod tests { } proptest! { - /// Any well-formed name round-trips: formatting a kind, port, link, - /// vlan tuple and parsing it back yields the same components. The - /// parser is purely syntactic, so the full u8/u16 ranges are exercised. + /// Any well-formed name round-trips: formatting a `kind`, `port`, + /// `link`, `vlan` tuple and parsing it back yields the same components. + /// The parser is purely syntactic, so the full u8/u16 ranges are + /// exercised. #[test] fn prop_roundtrip( is_rear in any::(), @@ -227,7 +249,7 @@ mod tests { link in any::(), vlan in proptest::option::of(any::()), ) { - let kind = if is_rear { Rear } else { Qsfp }; + let kind = if is_rear { TfportKind::Rear } else { TfportKind::Qsfp }; let mut ifname = format!("tfport{}{port}_{link}", kind.token()); if let Some(vlan) = vlan { ifname.push_str(&format!(".{vlan}")); diff --git a/mg-lower/src/ddm.rs b/mg-lower/src/ddm.rs index b8af84b0b..efa9e6d90 100644 --- a/mg-lower/src/ddm.rs +++ b/mg-lower/src/ddm.rs @@ -8,7 +8,10 @@ use ddm_admin_client::Client; use ddm_api_types_versions::latest::net::{MulticastOrigin, TunnelOrigin}; use oxnet::Ipv6Net; use slog::Logger; -use std::{net::Ipv6Addr, sync::Arc}; +use std::{ + net::{Ipv6Addr, SocketAddr}, + sync::Arc, +}; use crate::platform::Ddm; @@ -107,9 +110,19 @@ pub(crate) fn remove_tunnel_routes<'a, I: Iterator>( } } +/// Create a new DDM admin client. +/// +/// In production the lower half runs in the same zone as DDM, so `addr` is +/// `None` and the client targets the default `localhost:8000`. Tests pass an +/// explicit `addr` to reach a DDM listening elsewhere (for example a +/// dynamically assigned port in an integration harness). #[cfg(target_os = "illumos")] -pub fn new_ddm_client(log: &Logger) -> Client { - Client::new("http://localhost:8000", log.clone()) +pub fn new_ddm_client(log: &Logger, addr: Option) -> Client { + let host = match addr { + Some(addr) => format!("http://{addr}"), + None => "http://localhost:8000".to_string(), + }; + Client::new(&host, log.clone()) } pub(crate) fn add_multicast_routes< diff --git a/mg-lower/src/dendrite.rs b/mg-lower/src/dendrite.rs index da7e1ab11..6eae54753 100644 --- a/mg-lower/src/dendrite.rs +++ b/mg-lower/src/dendrite.rs @@ -12,13 +12,12 @@ use dpd_client::Client as DpdClient; use dpd_client::types::{self, LinkState, Route}; use mg_api_types::rdb::path::Path; use mg_api_types::rdb::prefix::Prefix; -use mg_common::tfport::{parse_tfport_name, tfport_port_id}; use oxnet::{IpNet, Ipv4Net, Ipv6Net}; use slog::Logger; use std::{ collections::{BTreeSet, HashSet}, hash::Hash, - net::{IpAddr, Ipv4Addr, Ipv6Addr}, + net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}, sync::Arc, time::Duration, }; @@ -340,12 +339,7 @@ where pub(crate) fn port_link_from_ifname( ifname: &str, ) -> Result<(types::PortId, types::LinkId), Error> { - let tfport = parse_tfport_name(ifname).map_err(Error::Tfport)?; - let port_id = - tfport_port_id(tfport.kind, tfport.port).map_err(Error::Tfport)?; - // TODO breakout considerations - let link_id = types::LinkId(tfport.link); - Ok((port_id, link_id)) + mg_common::tfport::port_link_from_ifname(ifname).map_err(Error::Tfport) } fn get_port_and_link( @@ -494,16 +488,21 @@ pub(crate) fn get_routes_for_prefix( Ok(result.into_iter().collect()) } -/// Create a new Dendrite/dpd client. The lower half always runs on the same -/// host/zone as the underlying platform. +/// Create a new Dendrite (DPD) client. +/// +/// In production the lower half runs in the same zone as DPD, so `addr` is +/// `None` and the client targets `localhost` on the default DPD port. Tests +/// pass an explicit `addr` to reach a DPD listening elsewhere (for example a +/// dynamically assigned port in an integration harness). #[cfg(target_os = "illumos")] -pub fn new_dpd_client(log: &Logger) -> DpdClient { +pub fn new_dpd_client(log: &Logger, addr: Option) -> DpdClient { let client_state = dpd_client::ClientState { tag: MG_LOWER_TAG.into(), log: log.clone(), }; - DpdClient::new( - &format!("http://localhost:{}", dpd_client::default_port()), - client_state, - ) + let host = match addr { + Some(addr) => format!("http://{addr}"), + None => format!("http://localhost:{}", dpd_client::default_port()), + }; + DpdClient::new(&host, client_state) } diff --git a/mg-lower/src/mrib.rs b/mg-lower/src/mrib.rs index 3121dcfc1..f61c46941 100644 --- a/mg-lower/src/mrib.rs +++ b/mg-lower/src/mrib.rs @@ -2,31 +2,35 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -//! MRIB (Multicast Routing Information Base) synchronization to DDM. +//! MRIB (Multicast Routing Information Base) lower-half synchronization. //! -//! This module watches for MRIB changes and propagates multicast group -//! subscriptions to DDM for distribution across the underlay network. +//! Advertises locally originated MRIB multicast groups to the DDM admin API, +//! which distributes them across the underlay to other sleds and racks. This is +//! the multicast analog of the unicast lower-half's tunnel-endpoint origination. +//! +//! Origination reads the local MRIB (`loc_mrib`) and is watch-driven, with the +//! periodic resync only as a backstop, the same shape as the unicast +//! lower-half's crate-level loop [`crate::run`]. +//! +//! The inbound membership half (resolving DDM-imported routes to switch +//! replication members in DPD) lives in ddmd (`ddm::mcast`), where both inputs, +//! the imported set and the peer table, are owned in-process. //! //! ## Data Flow //! //! ```text -//! MRIB (loc_mrib changes) -//! | -//! v [MribChangeNotification] -//! mg-lower/mrib.rs -//! | -//! v [MulticastOrigin] -//! DDM admin API -//! | -//! v [DDM exchange protocol] -//! Other sleds/racks +//! Origination (MRIB -> DDM -> underlay) +//! MRIB (loc_mrib changes) +//! | [MribChangeNotification] +//! v [MulticastOrigin] +//! DDM admin API --[DDM exchange]--> other sleds/racks //! ``` +//! +//! See RFD 488 for the multicast architecture. -use crate::ddm::{ - add_multicast_routes, new_ddm_client, remove_multicast_routes, -}; -use crate::platform::{Ddm, ProductionDdm}; -use ddm_api_types_versions::latest::net::{MulticastOrigin, Vni}; +use crate::ddm::{add_multicast_routes, remove_multicast_routes}; +use crate::platform::Ddm; +use ddm_api_types_versions::latest::net::MulticastOrigin; use rdb::Mrib; use rdb::types::{MribChangeNotification, MulticastAddr, MulticastRoute}; use slog::{Logger, debug, error, info}; @@ -36,58 +40,77 @@ use std::sync::mpsc::{RecvTimeoutError, channel}; use std::thread::sleep; use std::time::Duration; -const MG_LOWER_MRIB_TAG: &str = "mg-lower-mrib"; +pub(crate) const MG_LOWER_MRIB_TAG: &str = "mg-lower-mrib"; -/// Convert an MRIB [`MulticastRoute`] to a DDM [`MulticastOrigin`]. +/// Interval between periodic MRIB full syncs. /// -/// [`MulticastOrigin`]: ddm_admin_client::types::MulticastOrigin +/// Acts as a backstop in case an MRIB change notification is missed. Mirrors the +/// unicast lower-half's 1s resync cadence in `crate::run`. +const MRIB_PERIODIC_SYNC_INTERVAL: Duration = Duration::from_secs(1); + +/// Convert an MRIB `MulticastRoute` to a DDM `MulticastOrigin`. fn ddm_origin(route: &MulticastRoute) -> MulticastOrigin { MulticastOrigin { overlay_group: route.key.group().ip(), underlay_group: route.underlay_group, - vni: Vni::try_from(route.key.vni().as_u32()) - .expect("MRIB Vni is within Vni::MAX_VNI"), + vni: route.key.vni(), metric: 0, source: route.key.source(), } } -/// Run the MRIB synchronization loop. +/// Run the MRIB origination loop. /// -/// This function loops forever, watching for MRIB changes and synchronizing -/// them to DDM. It runs on the calling thread. -pub fn run(mrib: Mrib, log: Logger, rt: Arc) { +/// This function loops forever, watching for MRIB changes and advertising +/// locally originated multicast groups to DDM. +/// +/// It runs on the calling thread, so callers are responsible for running it in +/// a separate thread if asynchronous execution is required. +pub fn run( + mrib: Mrib, + log: Logger, + rt: Arc, + ddm: &impl Ddm, +) { loop { let (tx, rx) = channel(); // Register as MRIB watcher mrib.watch(MG_LOWER_MRIB_TAG.into(), tx); - let ddm = ProductionDdm { - client: new_ddm_client(&log), - }; - // Initial full sync - if let Err(e) = full_sync(&mrib, &ddm, &log, &rt) { + if let Err(e) = full_sync(&mrib, ddm, &log, &rt) { error!(log, "MRIB full sync failed: {e}"); info!(log, "restarting MRIB sync loop in one second"); + // Drop this iteration's watcher before retrying. The continue + // re-registers a fresh watcher, so without this the failed + // registration would accumulate in the MRIB watcher list on every + // retry. + mrib.unwatch(MG_LOWER_MRIB_TAG); + // Pause before retrying to keep a persistent failure from spinning. + // This is a backoff floor, kept independent of the resync cadence so + // tuning one does not silently change the other. + // + // Note: the unicast lower-half pauses in the same way + // (see `crate::run`). sleep(Duration::from_secs(1)); continue; } // Handle incremental changes loop { - match rx.recv_timeout(Duration::from_secs(10)) { + match rx.recv_timeout(MRIB_PERIODIC_SYNC_INTERVAL) { Ok(notification) => { if let Err(e) = - handle_change(&mrib, notification, &ddm, &log, &rt) + handle_change(&mrib, notification, ddm, &log, &rt) { error!(log, "MRIB change handling failed: {e}"); } } + // if we've not received updates in the timeout interval, do a + // full sync in case something has changed out from under us. Err(RecvTimeoutError::Timeout) => { - // Periodic full sync to catch any missed changes - if let Err(e) = full_sync(&mrib, &ddm, &log, &rt) { + if let Err(e) = full_sync(&mrib, ddm, &log, &rt) { error!(log, "MRIB periodic sync failed: {e}"); } } @@ -100,10 +123,10 @@ pub fn run(mrib: Mrib, log: Logger, rt: Arc) { } } -/// Perform a full synchronization of MRIB to DDM. +/// Perform a full synchronization of MRIB origination to DDM. /// -/// This compares the current MRIB loc_mrib with what DDM has advertised -/// and reconciles any differences. +/// Compares the current MRIB `loc_mrib` with what DDM has advertised and +/// reconciles any differences. pub(crate) fn full_sync( mrib: &Mrib, ddm: &D, @@ -170,14 +193,14 @@ fn handle_change( let mut to_remove = Vec::new(); for key in notification.changed { - // Check if route exists in loc_mrib (installed) + // Check if route exists in `loc_mrib` (installed) if let Some(route) = mrib.get_selected_route(&key) { let origin = ddm_origin(&route); if !ddm_current.contains(&origin) { to_add.push(origin); } } else { - // Route was removed from loc_mrib, so we need to find matching DDM + // Route is not in `loc_mrib`, so we need to find matching DDM // origin. We check all DDM origins to find any that match this key for ddm_origin in &ddm_current { // Reconstruct the key from the DDM origin to compare @@ -186,7 +209,7 @@ fn handle_change( && let Ok(ddm_key) = rdb::types::MulticastRouteKey::new( ddm_origin.source, overlay_group, - Vni::DEFAULT_MULTICAST_VNI, + ddm_origin.vni, ) && ddm_key == key { @@ -212,3 +235,222 @@ fn handle_change( Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::platform::test::TestDdm; + use rdb::test::get_test_db; + use rdb::types::{ + MulticastRouteKey, MulticastSourceProtocol, UnderlayMulticastIpv6, + }; + use std::net::Ipv6Addr; + + fn discard_logger() -> Logger { + Logger::root(slog::Discard, slog::o!()) + } + + /// Runtime handle for `full_sync`'s internal `block_on`. Tests run on a + /// plain thread (not a tokio worker), so blocking on this handle is safe. + fn runtime() -> (tokio::runtime::Runtime, Arc) { + let rt = tokio::runtime::Runtime::new().expect("build runtime"); + let handle = Arc::new(rt.handle().clone()); + (rt, handle) + } + + fn test_underlay() -> UnderlayMulticastIpv6 { + UnderlayMulticastIpv6::new(Ipv6Addr::new(0xff04, 0, 0, 0, 0, 0, 0, 1)) + .expect("valid underlay address") + } + + /// Build an any-source (*,G) static route for the given IPv4 group. + fn asm_route(a: u8, b: u8, c: u8, d: u8) -> MulticastRoute { + let group = MulticastAddr::new_v4(a, b, c, d).expect("valid group"); + let key = MulticastRouteKey::any_source(group); + MulticastRoute::new( + key, + test_underlay(), + MulticastSourceProtocol::Static, + ) + } + + #[test] + fn full_sync_advertises_groups_missing_from_ddm() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_full_sync_add", log.clone()).expect("db"); + let route = asm_route(225, 1, 1, 1); + db.add_static_mcast_routes(std::slice::from_ref(&route)) + .expect("add route"); + + let ddm = TestDdm::default(); + full_sync(db.mrib(), &ddm, &log, &handle).expect("full sync"); + + let originated = ddm.multicast_originated.lock().unwrap(); + assert_eq!(originated.len(), 1); + assert_eq!(originated[0], ddm_origin(&route)); + } + + #[test] + fn full_sync_withdraws_groups_absent_from_mrib() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_full_sync_remove", log.clone()).expect("db"); + + let stale = asm_route(225, 9, 9, 9); + let ddm = TestDdm::default(); + ddm.multicast_originated + .lock() + .unwrap() + .push(ddm_origin(&stale)); + + full_sync(db.mrib(), &ddm, &log, &handle).expect("full sync"); + + assert!(ddm.multicast_originated.lock().unwrap().is_empty()); + } + + #[test] + fn full_sync_in_sync_makes_no_changes() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_full_sync_noop", log.clone()).expect("db"); + let route = asm_route(225, 5, 5, 5); + db.add_static_mcast_routes(std::slice::from_ref(&route)) + .expect("add route"); + + let ddm = TestDdm::default(); + ddm.multicast_originated + .lock() + .unwrap() + .push(ddm_origin(&route)); + + full_sync(db.mrib(), &ddm, &log, &handle).expect("full sync"); + + let originated = ddm.multicast_originated.lock().unwrap(); + assert_eq!(originated.len(), 1, "in-sync group must not be re-added"); + assert_eq!(originated[0], ddm_origin(&route)); + } + + #[test] + fn full_sync_adds_and_withdraws_together() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_full_sync_mixed", log.clone()).expect("db"); + let keep = asm_route(225, 1, 1, 1); + db.add_static_mcast_routes(std::slice::from_ref(&keep)) + .expect("add route"); + + let ddm = TestDdm::default(); + let stale = asm_route(225, 2, 2, 2); + ddm.multicast_originated + .lock() + .unwrap() + .push(ddm_origin(&stale)); + + full_sync(db.mrib(), &ddm, &log, &handle).expect("full sync"); + + let originated = ddm.multicast_originated.lock().unwrap(); + assert_eq!(originated.len(), 1); + assert_eq!(originated[0], ddm_origin(&keep)); + } + + #[test] + fn handle_change_advertises_newly_installed_group() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_change_add", log.clone()).expect("db"); + let route = asm_route(225, 1, 1, 1); + db.add_static_mcast_routes(std::slice::from_ref(&route)) + .expect("add route"); + + let ddm = TestDdm::default(); + let notification = MribChangeNotification::from(route.key); + handle_change(db.mrib(), notification, &ddm, &log, &handle) + .expect("handle change"); + + let originated = ddm.multicast_originated.lock().unwrap(); + assert_eq!(originated.len(), 1); + assert_eq!(originated[0], ddm_origin(&route)); + } + + #[test] + fn handle_change_withdraws_removed_group() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_change_remove", log.clone()).expect("db"); + + // The group is advertised by DDM but never installed in the MRIB, + // modeling a route that was removed from `loc_mrib`. + let removed = asm_route(225, 9, 9, 9); + let ddm = TestDdm::default(); + ddm.multicast_originated + .lock() + .unwrap() + .push(ddm_origin(&removed)); + + let notification = MribChangeNotification::from(removed.key); + handle_change(db.mrib(), notification, &ddm, &log, &handle) + .expect("handle change"); + + assert!(ddm.multicast_originated.lock().unwrap().is_empty()); + } + + #[test] + fn handle_change_is_noop_when_already_advertised() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_change_noop", log.clone()).expect("db"); + let route = asm_route(225, 5, 5, 5); + db.add_static_mcast_routes(std::slice::from_ref(&route)) + .expect("add route"); + + let ddm = TestDdm::default(); + ddm.multicast_originated + .lock() + .unwrap() + .push(ddm_origin(&route)); + + let notification = MribChangeNotification::from(route.key); + handle_change(db.mrib(), notification, &ddm, &log, &handle) + .expect("handle change"); + + let originated = ddm.multicast_originated.lock().unwrap(); + assert_eq!( + originated.len(), + 1, + "already-advertised group must not be re-added" + ); + assert_eq!(originated[0], ddm_origin(&route)); + } + + #[test] + fn handle_change_withdraws_only_the_matching_group() { + let (_rt, handle) = runtime(); + let log = discard_logger(); + let db = get_test_db("mrib_change_selective", log.clone()).expect("db"); + + // DDM advertises two groups; only one is named in the change set and is + // absent from the MRIB, so only that one must be withdrawn. Exercises + // the key-reconstruction match in the removal path. + let removed = asm_route(225, 1, 1, 1); + let other = asm_route(225, 2, 2, 2); + let ddm = TestDdm::default(); + { + let mut originated = ddm.multicast_originated.lock().unwrap(); + originated.push(ddm_origin(&removed)); + originated.push(ddm_origin(&other)); + } + + let notification = MribChangeNotification::from(removed.key); + handle_change(db.mrib(), notification, &ddm, &log, &handle) + .expect("handle change"); + + let originated = ddm.multicast_originated.lock().unwrap(); + assert_eq!( + originated.len(), + 1, + "unrelated group must remain advertised" + ); + assert_eq!(originated[0], ddm_origin(&other)); + } +} diff --git a/mg-lower/src/platform.rs b/mg-lower/src/platform.rs index bc68ee94c..e56473758 100644 --- a/mg-lower/src/platform.rs +++ b/mg-lower/src/platform.rs @@ -1,4 +1,4 @@ -//! This crate contains traits that decouple mg-lower from the underlying +//! This module contains traits that decouple mg-lower from the underlying //! platform. This is useful for testing mg-lower while not having to //! have a running dpd, ddmd, or switch zone. //! @@ -271,6 +271,7 @@ pub trait SwitchZone { /// Production dpd trait that simply passes through calls to a dpd client. #[cfg(target_os = "illumos")] +#[derive(Clone)] pub struct ProductionDpd { pub client: DpdClient, } @@ -393,6 +394,7 @@ impl Dpd for ProductionDpd { /// Production ddm trait that simply passes through calls to a ddm client. #[cfg(target_os = "illumos")] +#[derive(Clone)] pub struct ProductionDdm { pub client: DdmClient, } diff --git a/mgd/src/main.rs b/mgd/src/main.rs index 4616b05de..c5a39cb71 100644 --- a/mgd/src/main.rs +++ b/mgd/src/main.rs @@ -104,6 +104,20 @@ struct RunArgs { /// SocketAddr for the BGP Dispatcher to listen on. #[arg(long, default_value = "[::]:179")] bgp_dispatcher_addr: SocketAddr, + + /// SocketAddr the Dendrite (DPD) API is listening on. When unset, the lower + /// half targets DPD at its default port on localhost (the co-located switch + /// zone). Set this to point the lower half at a DPD elsewhere, such as a + /// dynamically assigned port in an integration harness. + #[arg(long)] + dendrite_addr: Option, + + /// SocketAddr the DDM admin API is listening on. When unset, the lower half + /// targets DDM at its default localhost address (the co-located switch + /// zone). Set this to point the lower half at a DDM elsewhere, such as a + /// dynamically assigned port in an integration harness. + #[arg(long)] + ddm_addr: Option, } fn main() { @@ -157,24 +171,44 @@ async fn run(args: RunArgs) { #[cfg(all(feature = "mg-lower", target_os = "illumos"))] { - let rt = Arc::new(tokio::runtime::Handle::current()); - let ctx = context.clone(); - let log = log.clone(); - let db = ctx.db.clone(); - let stats = context.mg_lower_stats.clone(); let dpd = mg_lower::ProductionDpd { - client: mg_lower::new_dpd_client(&log), + client: mg_lower::new_dpd_client(&log, args.dendrite_addr), }; let ddm = mg_lower::ProductionDdm { - client: mg_lower::new_ddm_client(&log), + client: mg_lower::new_ddm_client(&log, args.ddm_addr), }; - let sw = mg_lower::ProductionSwitchZone {}; - Builder::new() - .name("mg-lower".to_string()) - .spawn(move || { - mg_lower::run(ctx.tep, db, log, stats, rt, &dpd, &ddm, &sw); - }) - .expect("failed to start mg-lower"); + + // Unicast lower-half: sync the unicast RIB to Dendrite. + { + let rt = Arc::new(tokio::runtime::Handle::current()); + let ctx = context.clone(); + let log = log.clone(); + let db = ctx.db.clone(); + let stats = context.mg_lower_stats.clone(); + let dpd = dpd.clone(); + let ddm = ddm.clone(); + let sw = mg_lower::ProductionSwitchZone {}; + Builder::new() + .name("mg-lower".to_string()) + .spawn(move || { + mg_lower::run(ctx.tep, db, log, stats, rt, &dpd, &ddm, &sw); + }) + .expect("failed to start mg-lower"); + } + + // Multicast lower-half: advertise locally originated MRIB groups to + // DDM. Underlay replication membership is reconciled in ddmd. + { + let rt = Arc::new(tokio::runtime::Handle::current()); + let log = log.clone(); + let mrib = context.db.mrib().clone(); + Builder::new() + .name("mg-lower-mrib".to_string()) + .spawn(move || { + mg_lower::mrib::run(mrib, log, rt, &ddm); + }) + .expect("failed to start mg-lower mrib sync"); + } } start_bgp_routers( diff --git a/multicast-types/src/lib.rs b/multicast-types/src/lib.rs index b1b5c78c8..45ca1c752 100644 --- a/multicast-types/src/lib.rs +++ b/multicast-types/src/lib.rs @@ -203,19 +203,4 @@ mod tests { Err(UnderlayMulticastError::InvalidIpv6(_)) )); } - - #[test] - fn from_str_rejects_non_admin_local() { - let result: Result = "ff0e::1".parse(); - assert!(matches!( - result, - Err(UnderlayMulticastError::NotInSubnet { .. }) - )); - } - - #[test] - fn from_str_accepts_admin_local() { - let parsed: UnderlayMulticastIpv6 = "ff04::1".parse().unwrap(); - assert_eq!(parsed.ip(), Ipv6Addr::new(0xff04, 0, 0, 0, 0, 0, 0, 1)); - } } From c49f755a3a7e64be974b4cc1ef1afc5d65adf53a Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Sun, 14 Jun 2026 15:54:58 +0000 Subject: [PATCH 12/16] [ddm] clarify --api-only peer-set docs, bind admin API synchronously Reword the peer-context and --api-only doc comments across the admin handler context, the multicast sweep, the discovery module, and ddmd startup to state that the peer set is empty when no state machines run. Start the admin Dropshot server synchronously so the API is bound before handler() returns, and log the bound address, which reflects the assigned port when 0 is requested. --- ddm/src/admin.rs | 40 +++++++++++++++++++++++++--------------- ddm/src/discovery/mod.rs | 7 +++---- ddm/src/mcast.rs | 6 ++++++ ddmd/src/main.rs | 10 +++++++--- 4 files changed, 41 insertions(+), 22 deletions(-) diff --git a/ddm/src/admin.rs b/ddm/src/admin.rs index c1b75f60c..1a370eeab 100644 --- a/ddm/src/admin.rs +++ b/ddm/src/admin.rs @@ -48,6 +48,11 @@ pub struct HandlerContext { pub event_channels: Vec>, pub db: Db, pub stats: Arc, + /// Per-interface state machine contexts shared with the multicast sweep, + /// seeded from the running state machines and read by the `/peers` view. + /// + /// Under the `--api-only` flag there are no state machines, so the set is + /// empty. pub peers: Vec, pub stats_handler: Arc>>>, pub log: Logger, @@ -85,7 +90,7 @@ pub fn handler( let api = api_description().map_err(|e| e.to_string())?; - let server = dropshot::ServerBuilder::new(api, context, ds_log) + let builder = dropshot::ServerBuilder::new(api, context, ds_log) .config(config) .version_policy(dropshot::VersionPolicy::Dynamic(Box::new( dropshot::ClientSpecifiesVersionInHeader::new( @@ -94,19 +99,24 @@ pub fn handler( ), ))); - info!(log, "admin: listening on {}", sa); + // Start synchronously so the admin API is bound before this function + // returns, and log the bound address, which reflects the assigned port + // when 0 is requested. + let server = match builder.start() { + Ok(server) => server, + Err(e) => { + error!(log, "admin: server start error {e:?}"); + return Ok(()); + } + }; + + info!(log, "admin: listening on {}", server.local_addr()); let log = log.clone(); spawn(async move { - match server.start() { - Ok(server) => { - info!(log, "admin: server started"); - match server.await { - Ok(()) => info!(log, "admin: server exited"), - Err(e) => error!(log, "admin: server error {:?}", e), - } - } - Err(e) => error!(log, "admin: server start error {:?}", e), + match server.await { + Ok(()) => info!(log, "admin: server exited"), + Err(e) => error!(log, "admin: server error {e:?}"), } }); @@ -474,11 +484,11 @@ pub fn api_description() ddm_admin_api_mod::api_description::() } -/// Snapshot the current peer table, keyed by interface index. +/// Snapshot the current peers, keyed by interface index. +/// +/// Reads the per-interface state machine contexts. /// -/// Reads per-interface state machines first and then layers in any -/// `--api-only` injected entries for interface indexes that are not -/// represented by a running state machine. +/// Under the `--api-only` flag there are no state machines, so the map is empty. pub(crate) fn do_get_peers( ctx: &Arc>, ) -> HashMap { diff --git a/ddm/src/discovery/mod.rs b/ddm/src/discovery/mod.rs index d397954a3..11c00300b 100644 --- a/ddm/src/discovery/mod.rs +++ b/ddm/src/discovery/mod.rs @@ -14,10 +14,9 @@ //! //! [`Version`] and [`DiscoveryError`] are platform-agnostic and stay in this //! module so the state machine type definitions in [`crate::sm`] continue to -//! compile when the routing runtime is gated out (e.g. Linux test fixtures -//! running `ddmd` with `--api-only`). The runtime helpers that drive -//! the protocol over UDPv6 sockets live in the [`runtime`] submodule and -//! are illumos-only. +//! compile when the routing runtime is gated out (e.g. a non-illumos `ddmd` +//! running with `--api-only`). The runtime helpers that drive the protocol +//! over UDPv6 sockets live in the [`runtime`] submodule and are illumos-only. //! //! ## Protocol //! diff --git a/ddm/src/mcast.rs b/ddm/src/mcast.rs index c5883fd76..7f6d76cfb 100644 --- a/ddm/src/mcast.rs +++ b/ddm/src/mcast.rs @@ -112,6 +112,12 @@ const DPD_REQUEST_TIMEOUT: Duration = Duration::from_secs(3); /// withdrawn group's DPD members are confirmed empty, so a group leaves the set /// exactly once its drain is complete. A re-import re-adds it on the next pass /// since the control plane writes to the DB before sending its trigger. +/// +/// `peers` is the set of per-interface state machine contexts, fixed at +/// startup. Peer identity lives behind interior mutability, so each pass +/// resolves against whatever peers have been discovered when it reads. +/// +/// Under `--api-only` there are no state machines, so the set is empty. pub fn run( db: Db, peers: Vec, diff --git a/ddmd/src/main.rs b/ddmd/src/main.rs index f2a5f9078..90838969e 100644 --- a/ddmd/src/main.rs +++ b/ddmd/src/main.rs @@ -107,9 +107,9 @@ struct Arg { sled_uuid: Option, /// Serve only the admin API. Skips the routing state machine - /// (discovery, exchange, route synchronization), allowing test fixtures - /// to obtain a real `ddmd` admin endpoint without the kernel-level - /// networking the state machine requires. + /// (discovery, exchange, route synchronization), allowing a real `ddmd` + /// admin endpoint without the kernel-level networking the state machine + /// requires. /// /// Analogous to `mgd --no-bgp-dispatcher`. #[arg(long, default_value_t = false, conflicts_with = "addr")] @@ -163,6 +163,10 @@ async fn run() { termination_handler(db.clone(), dpd.clone(), rt.clone(), log.clone()); let router_stats = Arc::new(RouterStats::default()); + // Per-interface state machine contexts shared between the multicast sweep + // and the admin context, seeded from the running state machines. + // + // Under --api-only there are no state machines, so the set is empty. let peers: Vec = sms.iter().map(|x| x.ctx.clone()).collect(); start_mcast_sweep( From 77ff902f1804c5981a76f55ef1dfe337c0639154 Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Mon, 22 Jun 2026 22:28:22 +0000 Subject: [PATCH 13/16] [ddm] run the multicast sweep as a runtime task, add periodic exchange resync The membership sweep moves from a dedicated thread with a std mpsc channel to a task on the daemon's runtime with a bounded tokio channel. Triggers are wake hints only, so a try_send dropped on a full channel costs at most one reconcile interval, and the sweep selects between a trigger and a periodic tick that defers rather than bursts after a long pass. Exchange state gains a periodic resync pull. The initial pull is one-shot, so routes a neighbor originates after we pull it, late multicast group memberships in particular, would otherwise never be imported absent a push from that neighbor. Periodic pulls import without redistributing, and each router runs its own resync, so a transit re-flooding its peers on every periodic pull would churn in steady state for no benefit. The smf manifest default for admin_host moves from ::1 to :: so the admin API is reachable over the underlay rather than only from the local host, matching mgd's existing default. --- .github/buildomat/test-ddm-common.sh | 6 +- Cargo.lock | 1 - client-common/src/multicast.rs | 51 -- .../versions/src/multicast_support/db.rs | 3 +- ddm-protocol/src/v4.rs | 4 +- .../tests/output/ddm_v4_protocol.json | 2 +- ddm/src/admin.rs | 29 +- ddm/src/db.rs | 246 ++++++-- ddm/src/discovery/mod.rs | 21 +- ddm/src/discovery/runtime.rs | 97 ++-- ddm/src/exchange/mod.rs | 18 +- ddm/src/exchange/reconcile.rs | 262 +++++++++ ddm/src/exchange/runtime.rs | 431 ++++++++++---- ddm/src/lib.rs | 11 + ddm/src/mcast.rs | 539 ++++++++++-------- ddm/src/sm/mod.rs | 26 +- ddm/src/sm/state.rs | 299 +++++++--- ddmd/src/main.rs | 28 +- mg-common/Cargo.toml | 1 - rdb/src/db.rs | 13 - smf/ddm/manifest.xml | 2 +- tests/src/ddm.rs | 33 +- 22 files changed, 1462 insertions(+), 661 deletions(-) create mode 100644 ddm/src/exchange/reconcile.rs diff --git a/.github/buildomat/test-ddm-common.sh b/.github/buildomat/test-ddm-common.sh index aefbf0278..5a49ccca4 100755 --- a/.github/buildomat/test-ddm-common.sh +++ b/.github/buildomat/test-ddm-common.sh @@ -1,9 +1,9 @@ #!/bin/bash export MAGHEMITE_VERSION=`git rev-parse HEAD` -export SOFTNPU_VERSION=591c64bf9765b6ed7cd8615ceb8cf6f8d117bd28 -export SIDECAR_LITE_VERSION=a95b7a9f78c08125f4e34106f5c885c7e9f2e8d5 -export DENDRITE_VERSION=72461d3a6e4724fd33454836d3c9d93c393fd4e4 +export SOFTNPU_VERSION=284c6830722548714128e63ea04bcca78ee27154 +export SIDECAR_LITE_VERSION=6f3311e8acd7e7e95c167aab61188355a93afe72 +export DENDRITE_VERSION=ab6c1a4326abbcb9f98a459ae74da4995e6b41af function cleanup { pfexec chown -R `id -un`:`id -gn` . diff --git a/Cargo.lock b/Cargo.lock index d2d658742..106aecabf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3884,7 +3884,6 @@ dependencies = [ "dpd-client", "libc", "libnet", - "omicron-common", "oximeter", "oximeter-producer", "oxnet", diff --git a/client-common/src/multicast.rs b/client-common/src/multicast.rs index 6287f9dfd..b42e4d785 100644 --- a/client-common/src/multicast.rs +++ b/client-common/src/multicast.rs @@ -213,18 +213,6 @@ impl FromStr for UnderlayMulticastIpv6 { mod tests { use super::*; - #[test] - fn overlay_accepts_v4_and_v6_multicast() { - assert!(OverlayMulticast::new("233.252.0.1".parse().unwrap()).is_ok()); - assert!(OverlayMulticast::new("ff0e::1".parse().unwrap()).is_ok()); - } - - #[test] - fn overlay_rejects_unicast() { - assert!(OverlayMulticast::new("192.0.2.1".parse().unwrap()).is_err()); - assert!(OverlayMulticast::new("2001:db8::1".parse().unwrap()).is_err()); - } - #[test] fn overlay_serde_rejects_unicast() { let json = @@ -234,36 +222,6 @@ mod tests { assert!(result.is_err()); } - #[test] - fn underlay_valid_ff04() { - let addr = Ipv6Addr::new(0xff04, 0, 0, 0, 0, 0, 0, 1); - assert!(UnderlayMulticastIpv6::new(addr).is_ok()); - } - - #[test] - fn underlay_rejects_non_admin_local() { - // ff0e:: is global scope, not admin-local - let addr = Ipv6Addr::new(0xff0e, 0, 0, 0, 0, 0, 0, 1); - assert!(UnderlayMulticastIpv6::new(addr).is_err()); - } - - #[test] - fn underlay_rejects_unicast() { - let addr = Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 1); - assert!(UnderlayMulticastIpv6::new(addr).is_err()); - } - - #[test] - fn underlay_serde_round_trip() { - let addr = UnderlayMulticastIpv6::new(Ipv6Addr::new( - 0xff04, 0, 0, 0, 0, 0, 0, 42, - )) - .unwrap(); - let json = serde_json::to_string(&addr).unwrap(); - let back: UnderlayMulticastIpv6 = serde_json::from_str(&json).unwrap(); - assert_eq!(addr, back); - } - #[test] fn underlay_serde_rejects_invalid() { // ff0e::1 serialized as an Ipv6Addr, then deserialized as @@ -275,13 +233,4 @@ mod tests { serde_json::from_str(&json); assert!(result.is_err()); } - - #[test] - fn from_str_rejects_unparseable() { - let result: Result = "not-an-ip".parse(); - assert!(matches!( - result, - Err(UnderlayMulticastError::InvalidIpv6(_)) - )); - } } diff --git a/ddm-api-types/versions/src/multicast_support/db.rs b/ddm-api-types/versions/src/multicast_support/db.rs index b2c851577..4f21e3b39 100644 --- a/ddm-api-types/versions/src/multicast_support/db.rs +++ b/ddm-api-types/versions/src/multicast_support/db.rs @@ -92,7 +92,7 @@ pub struct PeerInfo { pub if_name: Option, } -/// Down-convert v3 `PeerInfo` to v2 `PeerInfo` by dropping `if_name`. +/// Downconvert v3 `PeerInfo` to v2 `PeerInfo` by dropping `if_name`. impl From for crate::v2::db::PeerInfo { fn from(p: PeerInfo) -> Self { Self { @@ -120,7 +120,6 @@ mod tests { // The path is excluded from a route's identity, so two routes sharing an // origin and nexthop but carrying different paths are equal. - // `HashSet::replace` relies on this to refresh a stored route's path. #[test] fn route_identity_excludes_path() { let base = MulticastRoute { diff --git a/ddm-protocol/src/v4.rs b/ddm-protocol/src/v4.rs index 0d6c31da8..a982ca369 100644 --- a/ddm-protocol/src/v4.rs +++ b/ddm-protocol/src/v4.rs @@ -361,7 +361,7 @@ mod test { tunnel: None, multicast: Some(multicast_update()), }; - // Down-convert to v3 for an older peer, then back. Multicast has no v3 + // Downconvert to v3 for an older peer, then back. Multicast has no v3 // representation, so the round trip drops it. let v3 = v3::Update::from(update); let back = Update::from(v3); @@ -387,7 +387,7 @@ mod test { } // A v4 update carrying all three halves must keep its underlay and tunnel - // halves intact when down-converted for older peers, while the multicast + // halves intact when downconverted for older peers, while the multicast // half (which has no v3 or v2 wire form) is dropped. #[test] fn mixed_update_down_conversion_preserves_underlay_and_tunnel() { diff --git a/ddm-protocol/tests/output/ddm_v4_protocol.json b/ddm-protocol/tests/output/ddm_v4_protocol.json index cb6a7dd7d..0b10c49bf 100644 --- a/ddm-protocol/tests/output/ddm_v4_protocol.json +++ b/ddm-protocol/tests/output/ddm_v4_protocol.json @@ -69,7 +69,7 @@ } }, "MulticastOrigin": { - "description": "Wire form of a multicast group origin.\n\nThe validated counterpart (`ddm_api_types::net::MulticastOrigin`) carries an `UnderlayMulticastIpv6` and a `Vni`. As a frozen wire type this form stays primitive, a plain `Ipv6Addr` for the underlay group and a plain `u32` for the VNI. Validation happens when converting into the rich type at the exchange boundary.", + "description": "Wire form of a multicast group origin.\n\nThe validated counterpart (`ddm_api_types::net::MulticastOrigin`) carries an `UnderlayMulticastIpv6` and a `Vni`. As a frozen wire type this form stays unvalidated, a plain `Ipv6Addr` for the underlay group and a plain `u32` for the VNI. Validation happens when converting into that counterpart at the exchange boundary.", "type": "object", "required": [ "overlay_group", diff --git a/ddm/src/admin.rs b/ddm/src/admin.rs index e779276d1..27610e2cc 100644 --- a/ddm/src/admin.rs +++ b/ddm/src/admin.rs @@ -385,12 +385,12 @@ impl DdmAdminApi for DdmAdminApiImpl { .map_err(|e| HttpError::for_internal_error(e.to_string()))?; for e in &ctx.event_channels { - e.send(Event::Admin(AdminEvent::Announce(PrefixSet::Multicast( - groups.clone(), - )))) - .map_err(|e| { - HttpError::for_internal_error(format!("admin event send: {e}")) - })?; + e.send(Event::Admin(AdminEvent::AnnounceMulticast(groups.clone()))) + .map_err(|e| { + HttpError::for_internal_error(format!( + "admin event send: {e}" + )) + })?; } Ok(HttpResponseUpdatedNoContent()) @@ -403,17 +403,22 @@ impl DdmAdminApi for DdmAdminApiImpl { let ctx = lock!(ctx.context()); let groups = request.into_inner(); slog::info!(ctx.log, "withdraw multicast groups: {groups:#?}"); + // The modification is applied before any event is enqueued, and each + // state machine revalidates reachability when it processes the + // event. An import racing this request cannot be withdrawn against + // stale state, since the revalidation reads post-modification database + // state. The modification is idempotent, so a client retry is safe. ctx.db .withdraw_mcast(&groups) .map_err(|e| HttpError::for_internal_error(e.to_string()))?; for e in &ctx.event_channels { - e.send(Event::Admin(AdminEvent::Withdraw(PrefixSet::Multicast( - groups.clone(), - )))) - .map_err(|e| { - HttpError::for_internal_error(format!("admin event send: {e}")) - })?; + e.send(Event::Admin(AdminEvent::WithdrawMulticast(groups.clone()))) + .map_err(|e| { + HttpError::for_internal_error(format!( + "admin event send: {e}" + )) + })?; } Ok(HttpResponseUpdatedNoContent()) diff --git a/ddm/src/db.rs b/ddm/src/db.rs index eed37f298..ea0327c7a 100644 --- a/ddm/src/db.rs +++ b/ddm/src/db.rs @@ -81,6 +81,7 @@ impl Db { log, }) } + pub fn dump(&self) -> DbData { lock!(self.data).clone() } @@ -110,15 +111,13 @@ impl Db { } /// Underlay groups imported via `nexthop`, deduplicated. - /// - /// Filters under the lock and returns only the distinct group addresses, so - /// the caller never clones the full imported set just to keep one peer's - /// routes. This is the non-destructive analog of the next-hop filter in - /// [`Db::remove_nexthop_routes`]. pub fn mcast_groups_for_nexthop( &self, nexthop: Ipv6Addr, ) -> HashSet { + // Filter under the lock so the caller never clones the full imported + // set just to keep one peer's routes. Non-destructive analog of the + // next-hop filter in `remove_nexthop_routes`. lock!(self.data) .imported_mcast .iter() @@ -150,23 +149,58 @@ impl Db { } /// Atomically import and delete multicast routes under a single lock, - /// returning the effective [`McastRibDelta`] against the state before any - /// mutation. + /// returning the effective [`McastRibDelta`] against the state before + /// any modification. /// - /// This avoids a TOCTOU race where concurrent mutations between separate - /// lock acquisitions could produce an incorrect view difference. + /// The single lock avoids a TOCTOU race where concurrent modifications + /// between separate lock acquisitions could produce an incorrect delta. + /// Callers that redistribute the update also need post-modification + /// reachability and use + /// [`Db::update_imported_mcast_with_reachability`] instead. pub fn update_imported_mcast( &self, import: &HashSet, remove: &HashSet, ) -> McastRibDelta { - let mut data = lock!(self.data); + Self::apply_imported_mcast(&mut lock!(self.data), import, remove) + } + /// [`Db::update_imported_mcast`] variant that also captures a + /// [`MulticastReachability`] snapshot of post-modification reachability. + /// + /// The imported set is captured under the same lock as the modification, + /// so downstream withdrawal reconciliation cannot observe imported state + /// older than the modification that produced it. Callers that do not + /// redistribute have no reconciliation to feed and skip this variant's + /// imported-set clone and persistent origin read. + pub fn update_imported_mcast_with_reachability( + &self, + import: &HashSet, + remove: &HashSet, + ) -> (McastRibDelta, MulticastReachability) { + let (delta, imported) = { + let mut data = lock!(self.data); + let delta = Self::apply_imported_mcast(&mut data, import, remove); + (delta, data.imported_mcast.clone()) + }; + + // Persistent origins are not touched by this method, so reading them + // outside the lock still yields a post-modification snapshot. The + // added tree scan is acceptable, sled caches the tree and it stays + // small. + (delta, self.reachability_snapshot(imported)) + } + + /// Apply `import` and `remove` to the imported multicast set under the + /// caller-held lock, returning the effective delta. + fn apply_imported_mcast( + data: &mut DbData, + import: &HashSet, + remove: &HashSet, + ) -> McastRibDelta { let before = data.imported_mcast.clone(); - // A re-import carries the route's latest path vector. Route identity - // (PartialEq/Hash) excludes the path, so `extend`/`insert` would keep - // the existing entry and retain a stale path. `replace` overwrites the - // stored route, so the newest path wins. + // Route identity excludes the path, so `insert` would keep a stale + // path on re-import. `replace` lets the newest path win. for x in import { data.imported_mcast.replace(x.clone()); } @@ -203,16 +237,18 @@ impl Db { Ok(()) } + /// Persist multicast origins for advertisement to peers. pub fn originate_mcast( &self, origins: &HashSet, ) -> Result<(), Error> { let tree = self.persistent_data.open_tree(MCAST_ORIGINATE)?; for o in origins { - // Key by the metric-excluded identity, store the full origin as the - // value. `MulticastOrigin` equality ignores `metric`, so keying by - // identity lets a re-origination with a changed metric overwrite the - // stored entry instead of leaving a stale one under the old metric. + // Key by the metric-excluded identity, storing the full origin as + // the value. `MulticastOrigin` equality ignores `metric`, so keying + // by identity lets a re-origination with a changed metric overwrite + // the stored entry instead of leaving a stale one under the old + // metric. tree.insert( o.identity_key()?.as_str(), serde_json::to_string(o)?.as_str(), @@ -222,9 +258,8 @@ impl Db { Ok(()) } - /// Scan a persistent origin tree, parsing each `(key, value)` pair with - /// `parse` and skipping entries that fail to read or parse. `kind` names the - /// entry kind for log context. + /// Scan a persistent origin tree with `parse`, skipping entries that + /// fail to read or parse. `kind` names the entry kind for log context. fn scan_origin_tree( &self, tree: &str, @@ -272,7 +307,7 @@ impl Db { pub fn originated_tunnel(&self) -> Result, Error> { self.scan_origin_tree(TUNNEL_ORIGINATE, "tunnel origin", |key, _v| { - Ok(serde_json::from_str(&String::from_utf8_lossy(key))?) + Ok(serde_json::from_slice(key)?) }) } @@ -285,10 +320,28 @@ impl Db { /// Each origin is keyed by its metric-excluded identity and stored as the /// value, so the current metric is read back from the value rather than the /// key. + /// + /// This iterates the tree directly rather than going through + /// [`Db::scan_origin_tree`], which skips entries that fail to read or + /// parse. Withdrawal reconciliation treats the result as the complete + /// origin set, so a silently skipped entry could become a false final + /// withdrawal. Here any per-entry failure fails the whole read. The + /// caller surfaces that by degrading the snapshot's origin set, and + /// reconciliation then drops the withdrawal rather than treating the + /// missing origins as truly gone. + /// + /// # Errors + /// + /// Returns an error if the tree cannot be opened or any entry fails to + /// read or parse. pub fn originated_mcast(&self) -> Result, Error> { - self.scan_origin_tree(MCAST_ORIGINATE, "mcast origin", |_key, value| { - Ok(serde_json::from_str(&String::from_utf8_lossy(value))?) - }) + let tree = self.persistent_data.open_tree(MCAST_ORIGINATE)?; + tree.iter() + .map(|item| { + let (_key, value) = item?; + Ok(serde_json::from_slice(&value)?) + }) + .collect() } pub fn originated_mcast_count(&self) -> Result { @@ -317,6 +370,14 @@ impl Db { Ok(()) } + /// Remove persisted multicast origins. + /// + /// State machines revalidate reachability at processing time via + /// [`Db::multicast_reachability`] rather than from a snapshot captured + /// here. + /// + /// The modification lands before any event is enqueued, so the + /// processing-time snapshot is guaranteed to reflect this removal. pub fn withdraw_mcast( &self, origins: &HashSet, @@ -331,47 +392,93 @@ impl Db { Ok(()) } + /// Capture current multicast reachability for processing-time + /// revalidation. An event is enqueued only after its modification completes, + /// so a snapshot taken while processing that event is post-modification. It + /// also observes any later modification, which lets a state machine avoid + /// acting on reachability that has since been restored. + pub fn multicast_reachability(&self) -> MulticastReachability { + // Same lock-ordering discipline as `update_imported_mcast`: clone + // `imported_mcast` under the data lock and read persistent origins + // after dropping it, since the two sources are not co-modified. + let imported = lock!(self.data).imported_mcast.clone(); + self.reachability_snapshot(imported) + } + pub fn remove_nexthop_routes( &self, nexthop: Ipv6Addr, ) -> RemovedNexthopRoutes { - let mut data = lock!(self.data); - // Routes are generally held in sets to prevent duplication and provide - // handy set-algebra operations. - let mut removed = HashSet::new(); - for x in &data.imported { - if x.nexthop == nexthop { - removed.insert(x.clone()); + let (removed, tnl_removed, mcast_removed, imported_mcast) = { + let mut data = lock!(self.data); + let mut removed = HashSet::new(); + for x in &data.imported { + if x.nexthop == nexthop { + removed.insert(x.clone()); + } + } + for x in &removed { + data.imported.remove(x); } - } - for x in &removed { - data.imported.remove(x); - } - let mut tnl_removed = HashSet::new(); - for x in &data.imported_tunnel { - if x.nexthop == nexthop { - tnl_removed.insert(*x); + let mut tnl_removed = HashSet::new(); + for x in &data.imported_tunnel { + if x.nexthop == nexthop { + tnl_removed.insert(*x); + } + } + for x in &tnl_removed { + data.imported_tunnel.remove(x); } - } - for x in &tnl_removed { - data.imported_tunnel.remove(x); - } - let mut mcast_removed = HashSet::new(); - for x in &data.imported_mcast { - if x.nexthop == nexthop { - mcast_removed.insert(x.clone()); + let mut mcast_removed = HashSet::new(); + for x in &data.imported_mcast { + if x.nexthop == nexthop { + mcast_removed.insert(x.clone()); + } } - } - for x in &mcast_removed { - data.imported_mcast.remove(x); - } + for x in &mcast_removed { + data.imported_mcast.remove(x); + } + + let imported_mcast = data.imported_mcast.clone(); + (removed, tnl_removed, mcast_removed, imported_mcast) + }; + // Persistent origins are not touched by this method, so reading them + // outside the lock still yields a post-modification snapshot. RemovedNexthopRoutes { underlay: removed, tunnel: tnl_removed, multicast: mcast_removed, + mcast_reachability: self.reachability_snapshot(imported_mcast), + } + } + + /// Build a post-modification reachability snapshot around an imported + /// set captured under the data lock, taking ownership so no further + /// clone is needed. A persistent origin read failure degrades + /// `originated` to the empty set and marks the snapshot, so consumers + /// know the empty origin set is a read failure rather than a real + /// absence. + fn reachability_snapshot( + &self, + imported: HashSet, + ) -> MulticastReachability { + match self.originated_mcast() { + Ok(originated) => MulticastReachability { + imported, + originated, + origins_degraded: false, + }, + Err(e) => { + error!(self.log, "read remaining multicast origins: {e}"); + MulticastReachability { + imported, + originated: HashSet::new(), + origins_degraded: true, + } + } } } @@ -396,6 +503,41 @@ pub struct RemovedNexthopRoutes { pub underlay: HashSet, pub tunnel: HashSet, pub multicast: HashSet, + /// Post-modification multicast reachability snapshot captured under the same + /// removal that produced `multicast`. Downstream reconciliation reads + /// viable paths from here rather than reading the database again. + pub mcast_reachability: MulticastReachability, +} + +/// A snapshot of multicast reachability: imported routes and local origins. +/// +/// Only `Db` methods construct this. A snapshot is either captured by the +/// modification that produced a set of withdrawals or read at processing time +/// via [`Db::multicast_reachability`]. In both cases, it describes state no +/// older than that modification, since the event that triggers a +/// processing-time read is enqueued only after the modification completes. +#[derive(Debug, Clone)] +pub struct MulticastReachability { + imported: HashSet, + originated: HashSet, + origins_degraded: bool, +} + +impl MulticastReachability { + pub fn imported(&self) -> &HashSet { + &self.imported + } + + pub fn originated(&self) -> &HashSet { + &self.originated + } + + /// Whether the persistent origin read failed, degrading `originated` to + /// the empty set. A degraded snapshot cannot distinguish a withdrawn + /// origin from an unread one. + pub fn origins_degraded(&self) -> bool { + self.origins_degraded + } } #[derive( diff --git a/ddm/src/discovery/mod.rs b/ddm/src/discovery/mod.rs index 11c00300b..bdccb0087 100644 --- a/ddm/src/discovery/mod.rs +++ b/ddm/src/discovery/mod.rs @@ -16,7 +16,7 @@ //! module so the state machine type definitions in [`crate::sm`] continue to //! compile when the routing runtime is gated out (e.g. a non-illumos `ddmd` //! running with `--api-only`). The runtime helpers that drive the protocol -//! over UDPv6 sockets live in the [`runtime`] submodule and are illumos-only. +//! over UDPv6 sockets live in the `runtime` submodule and are illumos-only. //! //! ## Protocol //! @@ -89,13 +89,13 @@ //! //! The first byte indicates the version. The second byte is a flags bitfield. //! The first position `S` indicates a solicitation. The second position `A` -//! indicates an advertisement. The third position `C` indicates V4 multicast -//! capability, advertised independently of the version byte so old peers that -//! ignore it still peer at the floor version. All other positions are reserved -//! for future use. The third byte indicates the kind of router. Current values -//! are 0 for a server router and 1 for a transit routers. The fourth byte is a -//! hostname length followed directly by a hostname of up to 255 bytes in -//! length. +//! indicates an advertisement. The third position `C` indicates DDMv4 +//! (multicast) capability, advertised independently of the version byte so old +//! peers that ignore it still peer at the floor version. All other positions +//! are reserved for future use. The third byte indicates the kind of router. +//! Current values are 0 for a server router and 1 for a transit routers. The +//! fourth byte is a hostname length followed directly by a hostname of up to +//! 255 bytes in length. use thiserror::Error; @@ -105,7 +105,10 @@ mod runtime; #[cfg(all(feature = "backend", target_os = "illumos"))] pub(crate) use runtime::handler; -#[derive(Debug, Copy, Clone, PartialEq, Eq)] +// Ordering follows the ascending discriminants, so version-dependent +// behavior can use range checks (`>= Version::V4`) that remain correct +// as newer versions are added. +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] #[repr(u8)] pub enum Version { V2 = 2, diff --git a/ddm/src/discovery/runtime.rs b/ddm/src/discovery/runtime.rs index 4f9566931..580eddb9c 100644 --- a/ddm/src/discovery/runtime.rs +++ b/ddm/src/discovery/runtime.rs @@ -29,7 +29,7 @@ const DDM_MADDR: Ipv6Addr = Ipv6Addr::new(0xff02, 0, 0, 0, 0, 0, 0, 0xdd); const DDM_PORT: u16 = 0xddd; const SOLICIT: u8 = 1; const ADVERTISE: u8 = 1 << 1; -// Advertises V4 multicast capability without raising the discovery version +// Advertises DDMv4 (multicast) capability without raising the discovery version // byte past what old peers accept (the byte stays at the V2 floor). Following // RFC 5492, peers ignore capability bits they do not understand, so this is // backward compatible. @@ -352,7 +352,8 @@ fn handle_solicitation( } } -/// Outcome of negotiating the session version from a peer's advertisement. +/// The outcome of negotiating the session version from a peer's advertisement. +#[derive(Debug, PartialEq)] enum NegotiatedVersion { /// A usable version was negotiated directly from the advertised byte and /// capability flags. @@ -365,15 +366,18 @@ enum NegotiatedVersion { /// Negotiate the session version from a peer's advertised version byte and /// capability flags. /// -/// The version byte is a backward-compatible floor (V2) that all deployed -/// peers accept. Capability rides flags rather than the byte: MCAST_CAPABLE -/// signals V4, so a peer at the floor that sets it negotiates V4 while older -/// peers that ignore the flag still session at the floor. A conforming peer -/// never advertises a byte above the known maximum. +/// The version byte is a backward-compatible floor ([`Version::V2`]) that all +/// deployed peers accept. Capability rides flags rather than the byte: +/// [`MCAST_CAPABLE`] signals [`Version::V4`], so a peer at the floor that sets +/// it negotiates V4 while older peers that ignore the flag still session at +/// the floor. A conforming peer never advertises a byte above the known +/// maximum. /// /// A byte outside the known range is therefore a malformed or incompatible /// peer, not a capability hint, so it is rejected rather than capped. See -/// RFC 5492 for the capability-negotiation model. +/// [RFC 5492] for the capability-negotiation model. +/// +/// [RFC 5492]: https://www.rfc-editor.org/rfc/rfc5492 fn negotiate_version(version: u8, flags: u8) -> NegotiatedVersion { let base = match version { 2 => Version::V2, @@ -443,9 +447,9 @@ fn handle_advertisement( stats.peer_address_changes.fetch_add(1, Ordering::Relaxed); } nbr.last_seen = Instant::now(); - let changed = nbr.version != version; + let version_changed = nbr.version != version; nbr.version = version; - changed + version_changed } None => { inf!( @@ -544,50 +548,39 @@ fn advertise( mod tests { use super::*; - // Without the capability flag the negotiated version follows the byte: the - // V2 floor and the in-range bytes map straight through. + // Pins the negotiation contract. + // + // Without the capability flag the byte is the floor and maps straight + // through, MCAST_CAPABLE raises any accepted byte to V4, and out-of-range + // bytes are rejected even with the flag set so a malformed advertisement + // cannot replace a valid neighbor. #[test] - fn floor_and_in_range_without_flag() { - assert!(matches!( - negotiate_version(2, 0), - NegotiatedVersion::Use(Version::V2) - )); - assert!(matches!( - negotiate_version(3, 0), - NegotiatedVersion::Use(Version::V3) - )); - assert!(matches!( - negotiate_version(4, 0), - NegotiatedVersion::Use(Version::V4) - )); - } - - // The MCAST_CAPABLE flag raises any accepted byte to V4. - #[test] - fn capability_flag_raises_to_v4() { - for version in [2u8, 3, 4] { - assert!(matches!( - negotiate_version(version, MCAST_CAPABLE), - NegotiatedVersion::Use(Version::V4) - )); - } - } - - // Bytes outside the known range are rejected even with the capability flag - // set, so a malformed advertisement cannot replace a valid neighbor. - #[test] - fn out_of_range_rejected() { - for (version, flags) in [ - (0u8, 0u8), - (1, 0), - (1, MCAST_CAPABLE), - (5, 0), - (255, MCAST_CAPABLE), - ] { - assert!(matches!( + fn negotiate_version_contract() { + use NegotiatedVersion::{Rejected, Use}; + + let cases = [ + // Byte alone is the floor. + (2u8, 0u8, Use(Version::V2)), + (3, 0, Use(Version::V3)), + (4, 0, Use(Version::V4)), + // Capability flag raises any accepted byte to V4. + (2, MCAST_CAPABLE, Use(Version::V4)), + (3, MCAST_CAPABLE, Use(Version::V4)), + (4, MCAST_CAPABLE, Use(Version::V4)), + // Out-of-range bytes are rejected, flag or not. + (0, 0, Rejected), + (1, 0, Rejected), + (1, MCAST_CAPABLE, Rejected), + (5, 0, Rejected), + (255, MCAST_CAPABLE, Rejected), + ]; + + for (version, flags, expected) in cases { + assert_eq!( negotiate_version(version, flags), - NegotiatedVersion::Rejected - )); + expected, + "negotiate_version({version}, {flags:#05b})" + ); } } } diff --git a/ddm/src/exchange/mod.rs b/ddm/src/exchange/mod.rs index 73fd054b9..e4f0e04c2 100644 --- a/ddm/src/exchange/mod.rs +++ b/ddm/src/exchange/mod.rs @@ -18,18 +18,25 @@ //! The wire types (`Update`, `UnderlayUpdate`, `TunnelUpdate`, //! `MulticastUpdate`, and their versioned counterparts) live in the //! [`ddm_protocol`] crate. The runtime helpers that drive the HTTP exchange -//! protocol and program forwarding state live in the [`runtime`] submodule and -//! are illumos-only, since they call into [`crate::sys`] to install routes. +//! protocol and program forwarding state live in the `runtime` submodule and +//! are illumos-only, since they call into `crate::sys` to install routes. use thiserror::Error; +#[cfg(any(test, all(feature = "backend", target_os = "illumos")))] +mod reconcile; + #[cfg(all(feature = "backend", target_os = "illumos"))] mod runtime; +#[cfg(all(feature = "backend", target_os = "illumos"))] +pub(crate) use reconcile::reconcile_multicast_withdrawals; + #[cfg(all(feature = "backend", target_os = "illumos"))] pub(crate) use runtime::{ - announce_multicast, announce_tunnel, announce_underlay, do_pull_v4, - handler, pull, withdraw_multicast, withdraw_tunnel, withdraw_underlay, + ExchangeHandle, UpdateMode, announce_multicast, announce_tunnel, + announce_underlay, do_pull_v4, handler, pull, withdraw_multicast, + withdraw_tunnel, withdraw_underlay, }; #[derive(Error, Debug)] @@ -46,6 +53,9 @@ pub enum ExchangeError { #[error("timeout error: {0}")] Timeout(#[from] tokio::time::error::Elapsed), + #[error("peer returned status {0}")] + Status(hyper::StatusCode), + #[error("json error: {0}")] SerdeJson(#[from] serde_json::Error), } diff --git a/ddm/src/exchange/reconcile.rs b/ddm/src/exchange/reconcile.rs new file mode 100644 index 000000000..cc2e67d6e --- /dev/null +++ b/ddm/src/exchange/reconcile.rs @@ -0,0 +1,262 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Side-effect-free reconciliation management shared by exchange +//! runtime paths, taking state in as arguments and returning updates +//! without database or network effects. + +use ddm_api_types::net::MulticastOrigin; +use ddm_protocol::v4::{ + MulticastPathHop, MulticastPathVector, MulticastUpdate, +}; + +/// Rewrite each withdrawal as a replacement announcement when another path to +/// the origin remains, and as a final withdrawal otherwise. +/// +/// Downstream peers keep exactly one route per `(origin, nexthop)`, with this +/// router as the nexthop, regardless of how many paths line up behind it. +/// Blindly forwarding a withdrawal would drop the peer's only route through +/// this router even when the origin is still reachable in another way. For each +/// withdrawal, this checks the local origins first, then the best entry in +/// the imported set. A remaining path produces an announcement with a +/// refreshed path vector. +/// +/// Only when nothing remains is the final withdrawal emitted with `local_hop` +/// appended. +/// +/// The reachability snapshot always describes post-modification state. Peer +/// expiry and renumber capture it in `remove_nexthop_routes`, exchange +/// updates in `update_imported_mcast`, and the admin withdraws read it at +/// processing time via `multicast_reachability`, which is still +/// post-modification because the event is enqueued only after the +/// modification lands. +pub(crate) fn reconcile_multicast_withdrawals<'a>( + withdrawals: impl IntoIterator, + reachability: &crate::db::MulticastReachability, + local_hop: &MulticastPathHop, +) -> MulticastUpdate { + let mut update = MulticastUpdate::default(); + + for withdrawal in withdrawals { + let replacement = MulticastOrigin::try_from(&withdrawal.origin) + .ok() + .and_then(|origin| { + if let Some(local_origin) = + reachability.originated().get(&origin) + { + return Some(MulticastPathVector { + origin: local_origin.into(), + path: vec![local_hop.clone()], + }); + } + + reachability + .imported() + .iter() + .filter(|route| route.origin == origin) + // Route identity guarantees one entry per nexthop for this + // origin. Address order is only a stable tie-breaker (it + // does not assign semantics to multicast metric). + .min_by_key(|route| route.nexthop) + .map(|route| { + let mut path = route.path.clone(); + path.push(local_hop.clone()); + MulticastPathVector { + origin: (&route.origin).into(), + path, + } + }) + }); + + match replacement { + Some(replacement) => { + update.announce.insert(replacement); + } + // A degraded snapshot cannot confirm the absence of a local + // origin, so a final withdrawal here could tear down a route to + // an origin that is still reachable. Dropping the withdrawal is + // the safe direction. The worst case is a transient stale route + // to an origin that really is gone, while a false withdrawal + // could remove a peer's only path through this router. The + // periodic exchange resync repairs the drift either way. + None if reachability.origins_degraded() => {} + None => { + update + .withdraw + .insert(withdrawal.with_hop(local_hop.clone())); + } + } + } + + update +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::{Db, MulticastReachability}; + use ddm_api_types::db::MulticastRoute; + use slog::Logger; + use std::collections::HashSet; + use std::net::Ipv6Addr; + use tempfile::TempDir; + + fn origin(metric: u64) -> MulticastOrigin { + serde_json::from_value(serde_json::json!({ + "overlay_group": "233.252.0.1", + "underlay_group": "ff04::1", + "vni": 77, + "metric": metric, + })) + .unwrap() + } + + fn hop(router_id: &str, last: u16) -> MulticastPathHop { + MulticastPathHop::new( + router_id.to_string(), + Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, last), + ) + } + + /// Build a `MulticastReachability` snapshot through `Db` persistence + /// rather than constructing it by hand, so tests exercise the same + /// snapshot path production uses. Any origins in `originated` are + /// persisted before the imported set is applied, so the captured + /// snapshot reflects both sources of reachability. + fn snapshot( + imported: HashSet, + originated: HashSet, + ) -> (TempDir, MulticastReachability) { + let dir = TempDir::new().unwrap(); + let log = Logger::root(slog::Discard, slog::o!()); + let db = Db::new(dir.path().to_str().unwrap(), log).unwrap(); + if !originated.is_empty() { + db.originate_mcast(&originated).unwrap(); + } + let (_delta, reachability) = db + .update_imported_mcast_with_reachability( + &imported, + &HashSet::new(), + ); + (dir, reachability) + } + + #[test] + fn remaining_import_replaces_withdrawal_and_refreshes_path() { + let origin = origin(10); + let withdrawn = MulticastPathVector { + origin: (&origin).into(), + path: vec![hop("withdrawn", 1)], + }; + let remaining_hop = hop("remaining", 2); + let imported = HashSet::from([MulticastRoute { + origin: origin.clone(), + nexthop: "fe80::2".parse().unwrap(), + path: vec![remaining_hop.clone()], + }]); + let local_hop = hop("local", 3); + let (_dir, remaining) = snapshot(imported, HashSet::new()); + + let update = reconcile_multicast_withdrawals( + [&withdrawn], + &remaining, + &local_hop, + ); + + assert!(update.withdraw.is_empty()); + let replacement = update.announce.iter().next().unwrap(); + assert_eq!(replacement.path, vec![remaining_hop, local_hop]); + } + + #[test] + fn remaining_local_origin_replaces_withdrawal() { + let origin = origin(10); + let withdrawn = MulticastPathVector { + origin: (&origin).into(), + path: vec![hop("withdrawn", 1)], + }; + let local_hop = hop("local", 3); + let (_dir, remaining) = + snapshot(HashSet::new(), HashSet::from([origin])); + + let update = reconcile_multicast_withdrawals( + [&withdrawn], + &remaining, + &local_hop, + ); + + assert!(update.withdraw.is_empty()); + let replacement = update.announce.iter().next().unwrap(); + assert_eq!(replacement.path, vec![local_hop]); + } + + #[test] + fn final_withdrawal_preserves_path_and_appends_local_hop() { + let origin = origin(10); + let withdrawn_hop = hop("withdrawn", 1); + let withdrawn = MulticastPathVector { + origin: (&origin).into(), + path: vec![withdrawn_hop.clone()], + }; + let local_hop = hop("local", 3); + let (_dir, remaining) = snapshot(HashSet::new(), HashSet::new()); + + let update = reconcile_multicast_withdrawals( + [&withdrawn], + &remaining, + &local_hop, + ); + + assert!(update.announce.is_empty()); + let forwarded = update.withdraw.iter().next().unwrap(); + assert_eq!(forwarded.path, vec![withdrawn_hop, local_hop]); + } + + /// The processing-time read path used by admin withdraw + /// revalidation must observe both persisted origins and the current + /// imported set. + #[test] + fn multicast_reachability_reads_current_imported_and_originated() { + let dir = TempDir::new().unwrap(); + let log = Logger::root(slog::Discard, slog::o!()); + let db = Db::new(dir.path().to_str().unwrap(), log).unwrap(); + + // Distinct overlay/underlay groups so identity-based equality on + // `MulticastOrigin` keeps the persisted and imported origins apart. + let local_origin: MulticastOrigin = + serde_json::from_value(serde_json::json!({ + "overlay_group": "233.252.0.1", + "underlay_group": "ff04::1", + "vni": 77, + "metric": 10, + })) + .unwrap(); + + let imported_origin: MulticastOrigin = + serde_json::from_value(serde_json::json!({ + "overlay_group": "233.252.0.2", + "underlay_group": "ff04::2", + "vni": 77, + "metric": 20, + })) + .unwrap(); + + let route = MulticastRoute { + origin: imported_origin, + nexthop: "fe80::2".parse().unwrap(), + path: vec![hop("remote", 2)], + }; + + db.originate_mcast(&HashSet::from([local_origin.clone()])) + .unwrap(); + db.update_imported_mcast( + &HashSet::from([route.clone()]), + &HashSet::new(), + ); + + let reachability = db.multicast_reachability(); + assert_eq!(reachability.imported(), &HashSet::from([route])); + assert_eq!(reachability.originated(), &HashSet::from([local_origin])); + } +} diff --git a/ddm/src/exchange/runtime.rs b/ddm/src/exchange/runtime.rs index 87f925e45..b36027186 100644 --- a/ddm/src/exchange/runtime.rs +++ b/ddm/src/exchange/runtime.rs @@ -7,7 +7,7 @@ //! plumbing that drains received updates into the local DB and the //! forwarding platform via [`crate::sys`]. illumos-only. -use super::ExchangeError; +use super::{ExchangeError, reconcile_multicast_withdrawals}; use crate::db::{Route, effective_route_set}; use crate::discovery::Version; use crate::sm::{Config, Event, PeerEvent, SmContext}; @@ -35,6 +35,7 @@ use http_body_util::BodyExt; use hyper::body::Bytes; use hyper_util::client::legacy::Client; use hyper_util::rt::TokioExecutor; +use mg_common::lock; use slog::{Logger, o}; use std::collections::HashSet; use std::net::{Ipv6Addr, SocketAddrV6}; @@ -46,6 +47,11 @@ use tokio::time::timeout; const UNIT_EXCHANGE_SERVER: &str = "exchange_server"; +/// Bound on an entire pull request, from dispatch through to reading the full +/// response. Pulls run on state machine threads, so a stalled peer must not +/// block event handling. +const PULL_TIMEOUT: Duration = Duration::from_millis(250); + #[derive(Clone)] pub struct HandlerContext { ctx: SmContext, @@ -53,6 +59,46 @@ pub struct HandlerContext { log: Logger, } +/// How an update's imported routes propagate beyond the local DB. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum UpdateMode { + /// Import into the local DB only. + ImportOnly, + /// Import and re-announce to this router's other peers. Only transit + /// routers act on this. Server routers treat it as [`Self::ImportOnly`]. + Redistribute, +} + +/// A handle to a running exchange server, pairing the server task with the +/// shared request context so the state machine can rebind the peer address +/// on renumber without restarting the server. +/// +/// A renumber occurs when a peer's link-local unicast address changes. +/// [`crate::discovery`] detects the change and re-advertises the neighbor +/// under the new address. The neighbor is still the same router, so the +/// exchange server keeps running and only the nexthop address it assigns to +/// imports changes. +pub struct ExchangeHandle { + thread: tokio::task::JoinHandle<()>, + context: Arc>, +} + +impl ExchangeHandle { + pub fn abort(&self) { + self.thread.abort(); + } + + /// Rebind the handler's peer address after a renumber. The handler + /// assigns this address as the nexthop on every route it imports, so it + /// must track the state machine's view of the peer or else, post-renumber + /// imports leak under the prior address. + pub fn renumber_peer(&self, peer: Ipv6Addr) { + // Safe to block: callers run on state machine threads, outside + // the runtime. + self.context.blocking_lock().peer = peer; + } +} + pub(crate) fn announce_underlay( ctx: &SmContext, config: Config, @@ -168,6 +214,21 @@ pub(crate) fn do_pull_v2( Ok(serde_json::from_slice(&body)?) } +fn require_success( + response: hyper::Response, +) -> Result, ExchangeError> { + if response.status().is_success() { + Ok(response) + } else { + Err(ExchangeError::Status(response.status())) + } +} + +/// Fetch a pull response body, accepting only successful HTTP responses. +/// +/// The status is checked before the body reaches a versioned decoder. This is +/// especially important for V4, whose optional response fields could otherwise +/// make a Dropshot JSON error look like an empty route set. fn do_pull_common( uri: String, rt: &Arc, @@ -182,23 +243,37 @@ fn do_pull_common( let resp = client.request(req); + // The timeout covers reading the body too, since a peer that stalls + // mid-response would otherwise block indefinitely. rt.block_on(async move { - let body = timeout(Duration::from_millis(250), resp) - .await?? - .into_body() - .collect() - .await? - .to_bytes(); - Ok(body) + timeout(PULL_TIMEOUT, async { + let resp = require_success(resp.await?)?; + Ok(resp.into_body().collect().await?.to_bytes()) + }) + .await? }) } +/// Pull the peer's routes and import them. +/// +/// When `mode` is [`UpdateMode::Redistribute`] and this router is a transit, +/// the imported set is also announced to the other peers. The initial pull on +/// entering the exchange state redistributes. The periodic pull imports only. +/// Each router runs its own periodic pull, so a route learned here still +/// reaches every router through that router's pull. Redistributing on every +/// cycle would only resend updates transit peers already hold in steady +/// state. +/// +/// A non-successful HTTP response aborts the pull before decoding or +/// reconciliation, leaving the routes previously imported from that peer +/// unchanged. pub(crate) fn pull( ctx: SmContext, addr: Ipv6Addr, version: Version, rt: Arc, log: Logger, + mode: UpdateMode, ) -> Result<(), ExchangeError> { let pr: PullResponse = match version { Version::V2 => { @@ -208,14 +283,64 @@ pub(crate) fn pull( Version::V4 => do_pull_v4(&ctx, &addr, &rt)?, }; - let update = Update::announce(pr); + // A multicast-capable peer's pull response carries its complete + // advertisable multicast set, so an imported multicast route from this + // peer that is absent from the response indicates a withdraw we missed. + // Therefore, we synthesize withdraws for the absentee vectors so that the + // periodic pull repairs subtractive as well as additive drift. + // + // Underlay and tunnel imports keep their push-only withdraw semantics. + // Multicast reconciliation matters more because a stale import holds a + // DPD replication member. Reconciliation applies only to peers that + // negotiated wire protocol version 4 or later, the first version to + // carry multicast. An earlier response has no multicast half, and its + // converted empty set must not be read as a full withdraw. + let mcast_withdraw: HashSet = if version < Version::V4 + { + HashSet::new() + } else { + let announced: HashSet = pr + .multicast + .iter() + .flatten() + .filter_map(|pv| MulticastOrigin::try_from(&pv.origin).ok()) + .collect(); + ctx.db + .imported_mcast() + .iter() + .filter(|route| { + route.nexthop == addr && !announced.contains(&route.origin) + }) + .map(|route| MulticastPathVector { + origin: (&route.origin).into(), + path: Vec::new(), + }) + .collect() + }; - let hctx = HandlerContext { + let mut update = Update::announce(pr); + if !mcast_withdraw.is_empty() { + dbg!( + log, + ctx.config.if_name, + "pull reconcile: withdrawing {} stale multicast routes", + mcast_withdraw.len(), + ); + match update.multicast.as_mut() { + Some(m) => m.withdraw = mcast_withdraw, + None => { + update.multicast = + Some(MulticastUpdate::withdraw(mcast_withdraw)) + } + } + } + + let handler = HandlerContext { ctx, peer: addr, - log: log.clone(), + log, }; - handle_update(&update, &hctx); + handle_update(&update, &handler, mode); Ok(()) } @@ -229,10 +354,12 @@ fn send_update( rt: Arc, log: Logger, ) -> Result<(), ExchangeError> { - // The current wire form is V4. Down-convert through consecutive versions - // when a peer negotiated an older protocol. A multicast-only update has no - // representation before V4, so the down-converted form can be empty. Skip - // the send in that case rather than emit an empty payload to an older peer. + // The update arrives in the latest wire form. Downconvert through + // consecutive versions when a peer negotiated an older protocol. + // Conversion drops content the peer's version cannot represent (multicast + // did not exist before V4, for example), so the downconverted form can + // be empty. We skip the send in that case rather than emit an empty + // payload. let (payload, path) = match version { Version::V2 => { let update = v2::Update::from(v3::Update::from(update)); @@ -276,23 +403,23 @@ fn send_update_common( let resp = client.request(req); + // A completed request only counts as delivered when the peer's handler + // reported success. Connection failures and error statuses must surface as + // errors so the state machine can expire the peer. rt.block_on(async move { - match timeout(Duration::from_millis(config.exchange_timeout), resp) - .await - { - Ok(_) => Ok(()), - Err(e) => { - err!( - log, - config.if_name, - "peer request timeout to {}: {}", - uri, - e, - ); - ctx.stats.update_send_fail.fetch_add(1, Ordering::Relaxed); - Err(e.into()) - } + let result: Result<(), ExchangeError> = async { + let resp = + timeout(Duration::from_millis(config.exchange_timeout), resp) + .await??; + require_success(resp)?; + Ok(()) + } + .await; + if let Err(e) = &result { + err!(log, config.if_name, "peer update to {uri} failed: {e}"); + ctx.stats.update_send_fail.fetch_add(1, Ordering::Relaxed); } + result }) } @@ -301,12 +428,13 @@ pub fn handler( addr: Ipv6Addr, peer: Ipv6Addr, log: Logger, -) -> Result, String> { +) -> Result { let context = Arc::new(Mutex::new(HandlerContext { ctx: ctx.clone(), log: log.clone(), peer, })); + let handler_ctx = Arc::clone(&context); let sa = SocketAddrV6::new(addr, ctx.config.exchange_port, 0, 0); @@ -315,8 +443,6 @@ pub fn handler( ..Default::default() }; - // TODO(#740): unify dropshot logger level handling with `mgd`, which - // runs its dropshot logger at the parent log level. let ds_log = ConfigLogging::StderrTerminal { level: ConfigLoggingLevel::Error, } @@ -342,7 +468,7 @@ pub fn handler( } })?; - Ok(ctx.rt.spawn(async move { + let thread = ctx.rt.spawn(async move { match server.start().await { Ok(_) => wrn!( log, @@ -356,7 +482,12 @@ pub fn handler( e ), } - })) + }); + + Ok(ExchangeHandle { + thread, + context: handler_ctx, + }) } pub fn api_description() -> Result< @@ -410,7 +541,7 @@ async fn push_handler_common( ) -> Result { let ctx = ctx.context().lock().await.clone(); tokio::task::spawn_blocking(move || { - handle_update(&update, &ctx); + handle_update(&update, &ctx, UpdateMode::Redistribute); }) .await .map_err(|e| { @@ -620,12 +751,32 @@ async fn pull_handler_v4( })) } -fn handle_update(update: &Update, ctx: &HandlerContext) { +fn handle_update(update: &Update, ctx: &HandlerContext, mode: UpdateMode) { ctx.ctx .stats .updates_received .fetch_add(1, Ordering::Relaxed); + // Route application and peer cleanup take the same per-interface lock. + // This lets discovery publish identity and liveness changes without + // waiting for DPD, OPTE, or datastore work below. Once the lock is held, + // a brief identity check rejects an update whose peer has already expired + // or renumbered. Otherwise, the subsequent cleanup waits and removes + // anything this update imports. + let _route_update = lock!(ctx.ctx.iface.route_update); + let current_peer = lock!(ctx.ctx.iface.peer_identity) + .as_ref() + .map(|peer| peer.addr); + if current_peer != Some(ctx.peer) { + inf!( + ctx.log, + ctx.ctx.config.if_name, + "discarding update from stale peer {}", + ctx.peer, + ); + return; + } + if let Some(underlay_update) = &update.underlay { handle_underlay_update(underlay_update, ctx); } @@ -634,13 +785,25 @@ fn handle_update(update: &Update, ctx: &HandlerContext) { handle_tunnel_update(tunnel_update, ctx); } - if let Some(multicast_update) = &update.multicast { - handle_multicast_update(multicast_update, ctx); - } - - // distribute updates - - if ctx.ctx.config.kind == RouterKind::Transit { + // Only transit routers redistribute, so demote the mode on a server + // before it reaches the multicast handler. Only the redistribution path + // reconciles against a reachability snapshot, so only it pays for + // capturing one. + let mode = if ctx.ctx.config.kind == RouterKind::Transit { + mode + } else { + UpdateMode::ImportOnly + }; + let mcast_reachability = update + .multicast + .as_ref() + .and_then(|mu| handle_multicast_update(mu, ctx, mode)); + + // Event delivery from different interfaces is intentionally not globally + // ordered. A reversed pair can expose an older multicast view until the + // next successful V4 pull, whose complete response repairs missing and + // stale imports. This avoids a global lock on every multicast change. + if mode == UpdateMode::Redistribute { dbg!( ctx.log, ctx.ctx.config.if_name, @@ -654,43 +817,51 @@ fn handle_update(update: &Update, ctx: &HandlerContext) { .map(|update| update.with_path_element(ctx.ctx.hostname.clone())); // Multicast loop prevention is asymmetric with the underlay. The - // underlay relies on sender-side split-horizon, skipping any route - // whose nexthop is the destination peer. Multicast relies on - // receiver-side path-vector RPF, where, on receipt, any announcement - // already carrying our router_id is dropped. RPF is the authoritative - // loop guard and is strictly stronger than split-horizon because it - // catches loops of any length rather than only the immediate echo. + // underlay filters on send, skipping any route whose nexthop is the + // destination peer. Multicast drops, on receipt, any + // announcement whose path already carries our router_id. The path + // check is required because a replacement announcement goes to every + // peer, so a peer can appear mid-path rather than as the nexthop, + // and paths can cross several transits, forming loops longer than + // the immediate echo. // - // We apply that same RPF filter here before redistributing, dropping - // any path vector that already traversed us. Forwarding such a vector - // is harmless to a peer that already has us in its path (its own RPF - // drops it), but would propagate a looped path to a peer that does - // not, inflating its collection of path vectors. + // The same filter applies here before redistributing. A peer already + // in a vector's path would drop it anyway, but a peer that is not + // would import a looped path. let hostname = &ctx.ctx.hostname; - let multicast = update.multicast.as_ref().map(|update| { - let hop = - MulticastPathHop::new(hostname.clone(), ctx.ctx.config.addr); - let passes_rpf = |path_vector: &&MulticastPathVector| { - !path_vector - .path - .iter() - .any(|hop| &hop.router_id == hostname) - }; - MulticastUpdate { - announce: update - .announce - .iter() - .filter(passes_rpf) - .map(|path_vector| path_vector.with_hop(hop.clone())) - .collect(), - withdraw: update - .withdraw - .iter() - .filter(passes_rpf) - .map(|path_vector| path_vector.with_hop(hop.clone())) - .collect(), - } - }); + + // The snapshot came from the `handle_multicast_update` modification, so + // reconciliation reads state consistent with the local application. + let multicast = + update + .multicast + .as_ref() + .zip(mcast_reachability.as_ref()) + .map(|(update, reachability)| { + let hop = MulticastPathHop::new( + hostname.clone(), + ctx.ctx.config.addr, + ); + + let is_loop_free = |path_vector: &&MulticastPathVector| { + !path_vector + .path + .iter() + .any(|hop| &hop.router_id == hostname) + }; + + let mut reconciled = reconcile_multicast_withdrawals( + update.withdraw.iter().filter(is_loop_free), + reachability, + &hop, + ); + reconciled.announce.extend( + update.announce.iter().filter(is_loop_free).map( + |path_vector| path_vector.with_hop(hop.clone()), + ), + ); + reconciled + }); let push = Arc::new(Update { underlay, @@ -699,8 +870,15 @@ fn handle_update(update: &Update, ctx: &HandlerContext) { }); for ec in &ctx.ctx.event_channels { - ec.send(Event::Peer(PeerEvent::Push(Arc::clone(&push)))) - .unwrap(); + if let Err(e) = + ec.send(Event::Peer(PeerEvent::Push(Arc::clone(&push)))) + { + err!( + ctx.log, + ctx.ctx.config.if_name, + "deliver redistributed update: {e}", + ); + } } } } @@ -848,33 +1026,24 @@ fn handle_underlay_update(update: &v3::UnderlayUpdate, ctx: &HandlerContext) { .store(ctx.ctx.db.imported_count() as u64, Ordering::Relaxed); } -fn handle_multicast_update(update: &MulticastUpdate, ctx: &HandlerContext) { +fn handle_multicast_update( + update: &MulticastUpdate, + ctx: &HandlerContext, + mode: UpdateMode, +) -> Option { let db = &ctx.ctx.db; let hostname = &ctx.ctx.hostname; let mut import = HashSet::new(); + let mut remove = HashSet::new(); + // A replacement is broadcast to every peer, including peers already in + // its path. For such a peer, the looped announce implicitly invalidates + // its old route through the sender. A clean vector for the same + // `(origin, peer)` in this update takes precedence. for path_vector in &update.announce { - // Path-vector RPF: drop if our router_id appears in the path, - // indicating the announcement has already traversed us. - if path_vector - .path - .iter() - .any(|hop| &hop.router_id == hostname) - { - trc!( - ctx.log, - ctx.ctx.config.if_name, - "dropping multicast announce for {}; loop detected \ - (path length {})", - path_vector.origin.overlay_group, - path_vector.path.len(), - ); - continue; - } - // Promote the wire origin to the validated form. Peer-supplied routes // are otherwise trusted, but the underlay group reaches DPD directly, - // so promotion enforces its ff04::/64 invariant (and the VNI range) + // so a promotion enforces its ff04::/64 invariant (and the VNI range) // before the route can be stored. An invalid origin is dropped rather // than tracking a group DPD would refuse to program. let origin = match MulticastOrigin::try_from(&path_vector.origin) { @@ -890,18 +1059,39 @@ fn handle_multicast_update(update: &MulticastUpdate, ctx: &HandlerContext) { } }; - import.insert(MulticastRoute { + let route = MulticastRoute { origin, nexthop: ctx.peer, path: path_vector.path.clone(), - }); + }; + if path_vector + .path + .iter() + .any(|hop| &hop.router_id == hostname) + { + if !import.contains(&route) { + dbg!( + ctx.log, + ctx.ctx.config.if_name, + "removing multicast route for {} via {}; \ + looped announce (path length {})", + path_vector.origin.overlay_group, + ctx.peer, + path_vector.path.len(), + ); + remove.insert(route); + } + } else { + // This also cancels an implicit removal if a looped vector for + // the same route appeared earlier in the unordered announce set. + remove.remove(&route); + import.insert(route); + } } - let mut remove = HashSet::new(); for path_vector in &update.withdraw { - // Path-vector RPF applies symmetrically to withdraws: a withdraw whose - // path already contains this router's id is an echo of a local - // redistribution and must not be acted on. + // A withdrawal whose path already contains this router is an echo of + // a local redistribution and must not be acted on. if path_vector .path .iter() @@ -933,8 +1123,7 @@ fn handle_multicast_update(update: &MulticastUpdate, ctx: &HandlerContext) { } }; - // The empty path is safe, as MulticastRoute's PartialEq/Hash excludes - // the path field, so this matches by (origin, nexthop) only. + // Route identity is (origin, nexthop), so an empty path matches. remove.insert(MulticastRoute { origin, nexthop: ctx.peer, @@ -942,16 +1131,30 @@ fn handle_multicast_update(update: &MulticastUpdate, ctx: &HandlerContext) { }); } - // Atomic import + delete + diff under a single lock. - let delta = db.update_imported_mcast(&import, &remove); + // Atomic import + delete + diff under a single lock. The redistribution + // path also reconciles against a post-modification reachability snapshot, + // captured under that same lock scope. + let (delta, reachability) = match mode { + UpdateMode::Redistribute => { + let (delta, reachability) = + db.update_imported_mcast_with_reachability(&import, &remove); + (delta, Some(reachability)) + } + UpdateMode::ImportOnly => { + (db.update_imported_mcast(&import, &remove), None) + } + }; // Notify the multicast sweep of each affected underlay group so it - // reconciles the group's DPD members. Only the sweep writes to DPD; - // this handler records the import and signals. Deriving the notification + // reconciles the group's DPD members. Only the sweep writes to DPD. + // + // This handler records the import and signals, deriving the notification // from the effective diff rather than the requested sets avoids waking the // sweep for routes that were already present or already absent. crate::mcast::notify_affected_groups( delta.added.iter().chain(delta.removed.iter()), &ctx.ctx.mcast_notify, ); + + reachability } diff --git a/ddm/src/lib.rs b/ddm/src/lib.rs index ee6f27ad8..2bfb631e9 100644 --- a/ddm/src/lib.rs +++ b/ddm/src/lib.rs @@ -17,6 +17,17 @@ pub const COMPONENT_DDM: &str = "ddm"; pub const MOD_ADMIN: &str = "admin"; pub const MOD_EXCHANGE: &str = "exchange"; +/// Capacity of the channel carrying wake hints to the multicast sweep. +/// +/// Wake messages are best-effort: senders drop a hint when the channel is full, +/// and the sweep's periodic reconciliation of its full tracked set repairs the +/// omission. A depth of one would suffice semantically, while this small buffer +/// absorbs routine bursts. +/// +/// This constant lives outside the `backend`-feature-gated `mcast` module +/// because `ddmd` wires the channel in every supported feature configuration. +pub const MCAST_NOTIFY_CHANNEL_DEPTH: usize = 8; + /// Wrap a set in `Some`, treating an empty set as absence. /// /// # Returns diff --git a/ddm/src/mcast.rs b/ddm/src/mcast.rs index 7f6d76cfb..b4691570e 100644 --- a/ddm/src/mcast.rs +++ b/ddm/src/mcast.rs @@ -19,7 +19,7 @@ //! and peer expiry send the group's address down a notify channel to wake the //! sweep early. Peer-link resolution does the same for any import that raced //! ahead of the link. Absent a trigger, the sweep self-ticks on -//! [`RECONCILE_INTERVAL`], which is also the drift-repair backstop. The address +//! `RECONCILE_INTERVAL`, which bounds how long drift persists. The address //! on a trigger is only a wake hint: the sweep always reconciles the full set, //! so a coalesced or missed trigger costs at most one interval of latency. //! @@ -29,6 +29,8 @@ //! imports are withdrawn stays in the sweep until its DPD member list is //! confirmed empty, then drops out, so a withdrawn group is emptied exactly //! once and the tracked set stays bounded to active and recently active groups. +//! Discovering pre-existing DPD-only groups is deferred for a startup grace so +//! a restart does not empty groups whose imports have not yet been re-learned. //! //! DPD's only member-write surface is a full-list replace, so every member edit //! is a read-modify-write. Groups reconcile concurrently within a pass, but the @@ -66,23 +68,22 @@ use dpd_client::types::{ }; use dpd_client::{Client, ClientState}; use futures::TryStreamExt; -use futures::future::join_all; +use futures::stream::{self, StreamExt}; use mg_common::lock; use reqwest::StatusCode; -use slog::{Logger, debug, error, warn}; +use slog::{Logger, debug, error, info, warn}; use std::collections::{HashMap, HashSet}; use std::net::Ipv6Addr; -use std::sync::Arc; -use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender}; -use std::time::Duration; +use std::time::{Duration, Instant}; +use tokio::sync::mpsc::{Receiver, Sender}; /// Interval between the sweep's periodic membership reconcile passes. /// /// A trigger wakes the sweep immediately, so this interval governs only the -/// drift-repair backstop: how quickly drift and any change not delivered as a +/// periodic resync: how quickly drift and any change not delivered as a /// trigger converge into DPD. It is kept coarse to bound idle DPD churn, since /// membership changes themselves arrive as triggers. -const RECONCILE_INTERVAL: Duration = Duration::from_secs(10); +pub(crate) const RECONCILE_INTERVAL: Duration = Duration::from_secs(10); /// Per-request timeout for a single DPD member operation. /// @@ -91,39 +92,57 @@ const RECONCILE_INTERVAL: Duration = Duration::from_secs(10); /// expected DPD member operation latency so that it fires only on a genuine /// stall, and low enough that a group's sequential fetch-then-write pair /// (`2 * DPD_REQUEST_TIMEOUT`) stays under [`RECONCILE_INTERVAL`], so a single -/// stalled group cannot extend a pass beyond one backstop interval. This +/// stalled group cannot extend a pass beyond one reconcile interval. This /// stall-detection threshold is reasoned about independently of the /// convergence cadence set by [`RECONCILE_INTERVAL`]. A timed-out operation is /// logged distinctly and retried on the next pass. const DPD_REQUEST_TIMEOUT: Duration = Duration::from_secs(3); +/// Sets how long after startup the sweep waits before discovering DPD-only +/// groups. +/// +/// A restart races the sweep against the exchange machinery: the first pass +/// runs before peers have re-advertised their subscriptions. Seeding the +/// tracked set from DPD immediately would make those groups look withdrawn and +/// drain them, cutting live replication until the imports return. Two +/// reconcile intervals cover peer discovery and the initial pulls. Imported +/// groups are still reconciled immediately, and a group withdrawn after +/// startup drains normally because it is already tracked. +const STARTUP_SEED_GRACE: Duration = RECONCILE_INTERVAL.saturating_mul(2); + +/// Cap on concurrently reconciling groups within a single pass. +/// +/// Bounds the in-flight DPD requests a pass can generate so a large tracked +/// set cannot flood DPD with an unbounded burst of GETs and PUTs. +const MAX_CONCURRENT_GROUP_RECONCILES: usize = 16; + /// Run the multicast membership sweep. /// -/// Loops forever on the calling thread, so callers run it in a dedicated thread. +/// This loops forever, so callers spawn it as a dedicated task on the runtime. /// -/// Tracks the set of active underlay groups and reconciles them on each pass. -/// `notify_rx` is a wake hint only: a trigger wakes the sweep early, and absent -/// a trigger it self-ticks on [`RECONCILE_INTERVAL`]. Every pass reconciles the +/// We track the set of active underlay groups and reconcile them on each pass. +/// `notify_rx` is a wake hint only. A trigger wakes the sweep early, and absent +/// a trigger it self-ticks on `RECONCILE_INTERVAL`. Every pass reconciles the /// full tracked set, so the group address carried by a trigger is not consulted /// and a coalesced trigger costs at most one interval of latency. /// /// The tracked set is the union of every currently imported group and any group /// still being drained. `reconcile_group` returns `false` only once a -/// withdrawn group's DPD members are confirmed empty, so a group leaves the set -/// exactly once its drain is complete. A re-import re-adds it on the next pass -/// since the control plane writes to the DB before sending its trigger. +/// withdrawn group's DPD members are confirmed empty, so that a group leaves +/// the set exactly once its drain is complete. A re-import re-adds it on the +/// next pass since the control plane writes to the DB before sending its +/// trigger. /// /// `peers` is the set of per-interface state machine contexts, fixed at /// startup. Peer identity lives behind interior mutability, so each pass /// resolves against whatever peers have been discovered when it reads. /// /// Under `--api-only` there are no state machines, so the set is empty. -pub fn run( +pub async fn run( db: Db, peers: Vec, dpd: DpdConfig, - rt: Arc, - notify_rx: Receiver, + mut notify_rx: Receiver, log: Logger, ) { let client_state = ClientState { @@ -133,14 +152,20 @@ pub fn run( // Build the inner HTTP client explicitly to bound each request at // DPD_REQUEST_TIMEOUT. The progenitor-generated dpd_client defaults to a // 15s connect and request timeout, which exceeds RECONCILE_INTERVAL. A - // single stalled GET could outlast the whole backstop interval, and a + // single stalled GET could outlast the whole reconcile interval, and a // sequential fetch-then-write pair could run three times it. Stepping down // to DPD_REQUEST_TIMEOUT keeps a stalled group's pair under one interval. - let http = reqwest::ClientBuilder::new() + let http = match reqwest::ClientBuilder::new() .connect_timeout(DPD_REQUEST_TIMEOUT) .timeout(DPD_REQUEST_TIMEOUT) .build() - .expect("failed to build DPD HTTP client"); + { + Ok(http) => http, + Err(e) => { + error!(log, "failed to build DPD HTTP client, stopping sweep: {e}"); + return; + } + }; let client = Client::new_with_client( &format!("http://{}:{}", dpd.host, dpd.port), http, @@ -155,67 +180,81 @@ pub fn run( // On a fresh start, the imported set and triggers only reference groups // with live subscriptions, so a group whose imports were withdrawn while // `ddmd` was down would never re-enter the sweep and its stale replication - // members would persist. Folding those groups in once lets the first pass - // drain any that no peer still imports, while groups still imported simply - // reconcile as usual. - let mut tracked: HashSet = rt - .block_on(client.member_group_ips(&log)) - .into_iter() - .collect(); + // members would persist. After a startup grace, folding those groups in + // lets a pass drain any that no peer still imports, while groups still + // imported simply reconcile as usual. A failed listing is retried after + // another reconcile interval, since ordinary passes cannot discover + // orphans. + let mut tracked: HashSet = HashSet::new(); + let mut seeded = false; + let mut next_seed_attempt = Instant::now() + STARTUP_SEED_GRACE; loop { + if !seeded && Instant::now() >= next_seed_attempt { + match client.member_group_ips(&log).await { + Some(groups) => { + tracked.extend(groups); + seeded = true; + } + None => { + next_seed_attempt = Instant::now() + RECONCILE_INTERVAL; + } + } + } + // The imported set and resolved peer links are the same for every group // in a pass, so compute them once here rather than per group. let imported = db.imported_mcast(); let peer_links = resolve_peer_links(&peers, &log); - tracked = rt.block_on(reconcile_pass( - tracked, imported, peer_links, &client, &log, - )); - - // Wait for a trigger or the backstop interval, whichever comes first, - // then drain any burst since the next pass reconciles everything - // regardless. - match notify_rx.recv_timeout(RECONCILE_INTERVAL) { - Ok(_) => while notify_rx.try_recv().is_ok() {}, - Err(RecvTimeoutError::Timeout) => {} - Err(RecvTimeoutError::Disconnected) => { - // Unreachable while `ddmd` runs: `main()` owns the original - // `notify_tx` and parks for the daemon's lifetime, so the - // channel cannot close even if every per-peer sender clone is - // torn down. We stop the sweep rather than spin on a closed - // channel if that invariant ever changes. - error!(log, "multicast notify channel closed, stopping sweep"); - break; - } + tracked = + reconcile_pass(tracked, imported, peer_links, &client, &log).await; + + // Wait for a trigger or one idle reconcile interval, whichever comes + // first, then drain any burst since the next pass reconciles + // everything. A fresh sleep is sufficient because every trigger also + // runs a full pass, and avoids catch-up timer semantics after a slow + // pass. Both arms are cancel-safe leaf futures with no `.await` in + // their bodies, so the sweep cannot futurelock. + tokio::select! { + trigger = notify_rx.recv() => match trigger { + Some(_) => while notify_rx.try_recv().is_ok() {}, + None => { + // Unreachable while `ddmd` runs: `main()` owns the original + // `notify_tx` and parks for the daemon's lifetime, so the + // channel cannot close even if every per-peer sender clone + // is torn down. We stop the sweep rather than spin on a + // closed channel if that invariant ever changes. + error!(log, "multicast notify channel closed, stopping sweep"); + break; + } + }, + _ = tokio::time::sleep(RECONCILE_INTERVAL) => {} } } } -/// Signal the multicast sweep to reconcile each distinct underlay group touched -/// by `routes`. +/// Signal the multicast sweep for each distinct underlay group in `routes`. /// -/// The route iterator may repeat a group many times, one entry per next hop. -/// The groups are deduplicated, so the sweep wakes once per affected group. The -/// import and withdraw paths share this so both wake the sweep the same way. +/// The route iterator may repeat a group for multiple next hops, so group +/// addresses are deduplicated before notification. pub(crate) fn notify_affected_groups<'a>( routes: impl IntoIterator, notify: &Sender, ) { - let affected: HashSet = routes + let groups = routes .into_iter() .map(|route| route.origin.underlay_group.ip()) .collect(); - notify_groups(affected, notify); + notify_groups(groups, notify); } /// Wake the multicast sweep once per group in `groups`. fn notify_groups(groups: HashSet, notify: &Sender) { for group in groups { - // Best-effort trigger to wake the multicast sweep. The sweep owns the - // receiver for the daemon's lifetime, so this send does not fail during - // normal operation. - let _ = notify.send(group); + // A full channel or closed receiver is harmless because triggers are + // wake hints and the periodic pass reconciles the full tracked set. + let _ = notify.try_send(group); } } @@ -223,7 +262,7 @@ fn notify_groups(groups: HashSet, notify: &Sender) { /// link resolves. /// /// A multicast import already wakes the sweep, but a route imported before the -/// peer link resolved cannot be programmed yet, so it waits out the backstop +/// peer link resolved cannot be programmed yet, so it waits out the reconcile /// interval. Waking the peer's groups on resolution closes that window. The /// imported set is read, not consumed, so this is the non-destructive analog of /// the [`Db::remove_nexthop_routes`] removal on peer expiry. @@ -240,15 +279,20 @@ pub(crate) fn notify_peer_groups( /// /// Folds every currently imported group into `tracked`, reconciles the whole /// set concurrently, and returns only the groups `reconcile_group` reports as -/// still active. A withdrawn group lingers for exactly one pass to empty its DPD -/// members, then drops out on the following pass. Re-importing a dropped group -/// re-adds it here, since Omicron writes to the DB before triggering the sweep. +/// still active. A withdrawn group lingers for exactly one pass to empty its +/// DPD members, then drops out on the following pass. Re-importing a dropped +/// group re-adds it here, since Omicron writes to the DB before triggering the +/// sweep. +/// +/// The per-group futures run concurrently on this task rather than being +/// spawned, capped at [`MAX_CONCURRENT_GROUP_RECONCILES`] in flight, so a +/// group whose DPD call stalls does not serialize the others behind it and a +/// large tracked set cannot flood DPD. /// -/// The per-group futures run concurrently on this task rather than spawned, so a -/// group whose DPD call stalls does not serialize the others behind it. The pass -/// still returns only once its slowest group completes, but each request is -/// bounded by [`DPD_REQUEST_TIMEOUT`], so a stall delays the pass by that bound -/// at most rather than blocking it indefinitely. +/// The pass still returns only once its slowest group completes. Each request +/// is bounded by [`DPD_REQUEST_TIMEOUT`], but stalled groups beyond the +/// concurrency cap execute in waves, so a pass is bounded by one timeout per +/// wave of stalled groups rather than one timeout overall. async fn reconcile_pass( mut tracked: HashSet, imported: HashSet, @@ -264,17 +308,18 @@ async fn reconcile_pass( // resolved links by reference. let imported = &imported; let peer_links = &peer_links; - let reconciled = join_all(tracked.into_iter().map(|group_ip| async move { - let keep = - reconcile_group(group_ip, imported, peer_links, client, log).await; - (group_ip, keep) - })) - .await; - - reconciled - .into_iter() - .filter_map(|(group_ip, keep)| keep.then_some(group_ip)) + stream::iter(tracked) + .map(|group_ip| async move { + ( + group_ip, + reconcile_group(group_ip, imported, peer_links, client, log) + .await, + ) + }) + .buffer_unordered(MAX_CONCURRENT_GROUP_RECONCILES) + .filter_map(|(group_ip, keep)| async move { keep.then_some(group_ip) }) .collect() + .await } /// Whether a DPD client error is a request timeout. @@ -287,26 +332,14 @@ fn is_timeout(e: &dpd_client::Error) -> bool { } /// Outcome of writing a group's member list to DPD. -#[derive(Clone)] +#[derive(Clone, Copy)] enum WriteOutcome { /// Members were written. Updated, - /// DPD no longer authorizes the write against the group's tag. - /// - /// The tag, owned by DPD, changed from the value read this pass. On an - /// active group a later pass reads the current tag and retries. On a - /// withdrawn group the group was reassigned, so `ddmd` abandons it rather - /// than retrying. - TagReassigned, - /// The group is absent from DPD. - Gone, - /// The write stalled past [`DPD_REQUEST_TIMEOUT`]. - /// - /// Distinguished from [`WriteOutcome::Failed`] so a genuine stall is - /// surfaced separately, though both retry the group on the next pass. - TimedOut, - /// The write failed for an unexpected, non-timeout reason. - Failed, + /// The group disappeared or its tag changed after the preceding read. + Stale, + /// The write failed and should be retried on the next pass. + Retry, } /// Outcome of reading a group's state from DPD. @@ -317,21 +350,11 @@ enum FetchOutcome { /// The group does not exist in DPD, either because Omicron has not created /// it yet or because it has been deleted. Absent, - /// The read stalled past [`DPD_REQUEST_TIMEOUT`]. - /// - /// Distinguished from [`FetchOutcome::ReadFailed`] so a genuine stall is - /// surfaced separately, though both keep the group tracked for retry. - TimedOut, - /// The read failed transiently for a non-timeout reason, so the group's - /// state is unknown this pass. - ReadFailed, + /// The read failed, so the group's state is unknown this pass. + Retry, } /// DPD group operations the reconcile loop depends on. -/// -/// Abstracted behind a trait so [`reconcile_group`]'s keep/drop logic can be -/// exercised against a mock, without a live DPD endpoint. The production -/// implementation is [`Client`]. trait GroupClient { /// Read an underlay group's current members and authorization tag. async fn fetch_group( @@ -349,20 +372,21 @@ trait GroupClient { members: Vec, ) -> WriteOutcome; - /// Underlay groups that currently have members programmed in DPD. + /// Underlay groups that currently have members programmed in DPD, or + /// `None` if the listing failed. /// - /// Read once at startup to seed the sweep's tracked set. `ddmd` is the sole - /// writer of underlay members on this switch, so every group returned was - /// programmed by `ddmd` (or a prior incarnation) and is safe to fold-in. A - /// failure returns an empty set. Orphan recovery then waits for the next - /// `ddmd` restart whose listing succeeds, since the periodic sweep - /// reconciles only tracked and imported groups and never re-lists DPD. - async fn member_group_ips(&self, log: &Logger) -> Vec; + /// Read after the startup grace to seed the sweep's tracked set. `ddmd` is + /// the sole writer of underlay members on this switch, so every group + /// returned was programmed by `ddmd`, possibly before a restart, and is + /// safe to fold in. A failed listing is retried later, since ordinary + /// passes reconcile only tracked and imported groups and cannot otherwise + /// discover orphans. + async fn member_group_ips(&self, log: &Logger) -> Option>; } impl GroupClient for Client { /// Distinguishes a group that is genuinely absent (`FetchOutcome::Absent`) - /// from one whose state could not be read (`FetchOutcome::ReadFailed`), so a + /// from one whose state could not be read (`FetchOutcome::Retry`), so a /// withdrawn group is not dropped from the sweep on a transient read failure /// before its members are confirmed drained. async fn fetch_group( @@ -387,6 +411,7 @@ impl GroupClient for Client { ); FetchOutcome::Absent } + // Surface a stalled read distinctly from other failures. The sweep // retries the group on its next pass regardless. Err(e) if is_timeout(&e) => { @@ -395,17 +420,17 @@ impl GroupClient for Client { "get of underlay group {group_ip} timed out after \ {DPD_REQUEST_TIMEOUT:?}, retrying next pass" ); - FetchOutcome::TimedOut + FetchOutcome::Retry } Err(e) => { - error!(log, "failed to get underlay group {group_ip}: {e}"); - FetchOutcome::ReadFailed + warn!(log, "failed to get underlay group {group_ip}: {e}"); + FetchOutcome::Retry } } } /// The expected races, a tag change (403) or a deleted group (404), are - /// returned as outcomes rather than logged, leaving the reaction to the + /// returned as outcomes rather than logged, leaving the handling to the /// caller. async fn write_members( &self, @@ -423,42 +448,45 @@ impl GroupClient for Client { "tag for underlay group {group_ip} is invalid, skipping \ update: {e}" ); - return WriteOutcome::Failed; + return WriteOutcome::Retry; } }; + let body = MulticastGroupUpdateUnderlayEntry { members }; match self .multicast_group_update_underlay(&underlay_ip, &tag, &body) .await { Ok(_) => WriteOutcome::Updated, - Err(e) if e.status() == Some(StatusCode::FORBIDDEN) => { - WriteOutcome::TagReassigned + Err(e) + if matches!( + e.status(), + Some(StatusCode::FORBIDDEN | StatusCode::NOT_FOUND) + ) => + { + WriteOutcome::Stale } - Err(e) if e.status() == Some(StatusCode::NOT_FOUND) => { - WriteOutcome::Gone - } - // Surface a stalled write distinctly from other failures. Treated as - // `WriteOutcome::Failed` so the sweep retries it on its next pass. + // Log a stalled write distinctly from other failures, while both + // share the same retry outcome. Err(e) if is_timeout(&e) => { warn!( log, "update of underlay group {group_ip} members timed out \ after {DPD_REQUEST_TIMEOUT:?}, retrying next pass" ); - WriteOutcome::TimedOut + WriteOutcome::Retry } Err(e) => { - error!( + warn!( log, "failed to update underlay group {group_ip} members: {e}" ); - WriteOutcome::Failed + WriteOutcome::Retry } } } - async fn member_group_ips(&self, log: &Logger) -> Vec { + async fn member_group_ips(&self, log: &Logger) -> Option> { let groups: Vec = match self .multicast_groups_list_stream(None) .try_collect() @@ -468,23 +496,26 @@ impl GroupClient for Client { Err(e) => { warn!( log, - "could not list multicast groups to seed sweep, relying \ - on imports and the periodic backstop: {e}" + "could not list multicast groups to seed sweep, retrying \ + later: {e}" ); - return Vec::new(); + return None; } }; - groups - .into_iter() - .filter_map(|group| match group { - dpd_client::types::MulticastGroupResponse::Underlay { - group_ip, - members, - .. - } if !members.is_empty() => Some(*group_ip), - _ => None, - }) - .collect() + + Some( + groups + .into_iter() + .filter_map(|group| match group { + dpd_client::types::MulticastGroupResponse::Underlay { + group_ip, + members, + .. + } if !members.is_empty() => Some(*group_ip), + _ => None, + }) + .collect(), + ) } } @@ -497,6 +528,15 @@ impl GroupClient for Client { /// A peer omitted here is seen by `group_members` as an unresolved next hop, so /// a transient resolution failure neither drops a previously programmed member /// nor blocks a newly resolved one. +/// +/// The map keys on the peer's link-local address alone, with no interface +/// scope, since an imported route's `nexthop` carries no interface either. +/// This relies on the rack deriving link-local addresses from EUI-64, which +/// makes them unique across links rather than only within one ([RFC 4007], +/// section 5). A duplicate address on distinct links would collapse to one +/// entry, so a collision is logged as a warning. +/// +/// [RFC 4007]: https://www.rfc-editor.org/rfc/rfc4007#section-5 fn resolve_peer_links( peers: &[SmContext], log: &Logger, @@ -515,9 +555,21 @@ fn resolve_peer_links( ); continue; } + match mg_common::tfport::port_link_from_ifname(&if_name) { Ok(port_link) => { - peer_links.insert(peer.addr, port_link); + if let Some(prev) = + peer_links.insert(peer.addr, port_link.clone()) + && prev != port_link + { + warn!( + log, + "peer link-local address {} resolves to multiple \ + switch links ({prev:?} and {port_link:?}), violating \ + EUI-64 uniqueness; keeping the latter", + peer.addr + ); + } } Err(e) => warn!( log, @@ -577,7 +629,7 @@ fn group_members( /// desired set, so the periodic resync repairs member drift. /// /// Returns `true` to keep the group tracked, either while it still has imports -/// (as the drift backstop) or whenever its DPD state could not be read this +/// (so resync repairs drift) or whenever its DPD state could not be read this /// pass, and `false` only once the group has no imports and its DPD member list /// is confirmed empty, so it drops out of the sweep. async fn reconcile_group( @@ -605,7 +657,7 @@ async fn reconcile_group( // members. A withdrawn group must not drop out here, or stale // replication would stay programmed until some later re-import tracked // it again. - FetchOutcome::TimedOut | FetchOutcome::ReadFailed => return true, + FetchOutcome::Retry => return true, }; if !has_imports { @@ -617,7 +669,7 @@ async fn reconcile_group( return match client.write_members(log, group_ip, &tag, Vec::new()).await { WriteOutcome::Updated => { - debug!( + info!( log, "emptied withdrawn underlay group {group_ip} members" ); @@ -625,9 +677,9 @@ async fn reconcile_group( } // Already gone, or recreated under a tag that is no longer what // we've seen. Either way `ddmd` no longer programs this group. - WriteOutcome::Gone | WriteOutcome::TagReassigned => false, + WriteOutcome::Stale => false, // Retry the empty on the next pass. - WriteOutcome::TimedOut | WriteOutcome::Failed => true, + WriteOutcome::Retry => true, }; } @@ -636,6 +688,12 @@ async fn reconcile_group( // resolution failure neither drops a previously programmed member nor // blocks adding a newly resolved one. With every next hop resolved, the // derived set replaces the current members. + // + // Fail open: preserve every current member, not only those of unresolved + // routes, since a member cannot be attributed to a next hop without + // resolving it. A stale member, even one unrelated to the unresolved next + // hop, persists until every next hop resolves. Extra replication is + // preferred over dropping a live member. let to_write = if has_unresolved { let merged = union_members(&members, &existing); if merged.len() > members.len() { @@ -655,20 +713,12 @@ async fn reconcile_group( if !members_eq(&existing, &to_write) { match client.write_members(log, group_ip, &tag, to_write).await { WriteOutcome::Updated => { - debug!(log, "updated underlay group {group_ip} members") + info!(log, "updated underlay group {group_ip} members") } - WriteOutcome::TagReassigned => warn!( - log, - "tag no longer authorizes underlay group {group_ip}, retrying \ - with a fresh read next pass" - ), - WriteOutcome::Gone - | WriteOutcome::Failed - | WriteOutcome::TimedOut => {} + WriteOutcome::Stale | WriteOutcome::Retry => {} } } - // Active group: keep it tracked so the backstop repairs any later drift. true } @@ -679,15 +729,25 @@ fn union_members( base: &[MulticastGroupMember], extra: &[MulticastGroupMember], ) -> Vec { - base.iter() - .chain(extra.iter().filter(|member| !base.contains(member))) - .cloned() - .collect() + let mut merged = base.to_vec(); + for member in extra { + if !merged.contains(member) { + merged.push(member.clone()); + } + } + merged } -/// Compare two multicast member lists for set equality, ignoring order. +/// Compare two multicast member lists for set equality, ignoring order and +/// duplicates. +/// +/// `ddmd` never writes duplicates, but the list read back from DPD is not +/// trusted to be duplicate-free. A length check alone would be fooled by +/// duplicates, e.g. `[A, A]` against `[A, B]`, skipping the write that would +/// repair the drift, so containment is checked in both directions. fn members_eq(a: &[MulticastGroupMember], b: &[MulticastGroupMember]) -> bool { - a.len() == b.len() && a.iter().all(|member| b.contains(member)) + a.iter().all(|member| b.contains(member)) + && b.iter().all(|member| a.contains(member)) } #[cfg(test)] @@ -757,7 +817,7 @@ mod tests { } #[test] - fn distinct_peers_on_same_link_collapse_to_one_member() { + fn same_link_peers_share_member() { let peer_a = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); let peer_b = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 2); let group = underlay(1); @@ -780,7 +840,7 @@ mod tests { } #[test] - fn unresolved_nexthop_yields_no_members_and_sets_flag() { + fn unresolved_nexthop_returns_empty_and_unresolved() { let peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); let group = underlay(7); @@ -795,7 +855,7 @@ mod tests { } #[test] - fn mixed_resolution_yields_resolved_members_and_sets_flag() { + fn mixed_resolution_returns_members_and_unresolved() { let resolved = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); let unresolved_peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 2); let group = underlay(3); @@ -871,11 +931,14 @@ mod tests { *self.fetch.lock().unwrap() = FetchOutcome::Found(tag.to_string(), members); } - self.write_outcome.clone() + self.write_outcome } - async fn member_group_ips(&self, _log: &Logger) -> Vec { - self.member_groups.clone() + async fn member_group_ips( + &self, + _log: &Logger, + ) -> Option> { + Some(self.member_groups.clone()) } } @@ -916,11 +979,10 @@ mod tests { } /// Drives the sweep's cross-pass carry-over invariant: an active group is - /// tracked, a withdraw lingers one pass to empty its members then drops, and - /// a re-import re-adds and reprograms it. This exercises the tracked-set - /// state machine that the run loop builds on. + /// tracked, a withdraw takes one pass to empty its members and drop it, and + /// a re-import adds and reconciles it again. #[test] - fn pass_drains_withdrawn_group_then_drops_and_readds_on_reimport() { + fn group_drains_and_readds_across_passes() { let peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); let group = underlay(1); let active = HashSet::from([route(peer, group)]); @@ -934,20 +996,20 @@ mod tests { WriteOutcome::Updated, ); - // Pass 1: imported and already in sync, so the group is tracked (no - // write occurrs). + // Pass 1: imported and already in sync, so the group is tracked without + // a write. let tracked = run_pass(HashSet::new(), &active, &peer_links, &mock); assert_eq!(tracked, HashSet::from([group])); assert!(mock.writes().is_empty()); - // Pass 2: withdrawn ~ the group carries over from pass 1, its members - // are emptied, and then it drops out of the tracked set. + // Pass 2: the withdrawn group carries over from pass 1, its members are + // emptied, and then it drops out of the tracked set. let tracked = run_pass(tracked, &withdrawn, &peer_links, &mock); assert!(tracked.is_empty()); assert_eq!(mock.writes(), vec![Vec::::new()]); - // Pass 3: re-imported ~ the dropped group is re-added and reprogrammed, - // since DPD now holds no members for it. + // Pass 3: the dropped group is re-imported and reprogrammed, since DPD + // now holds no members for it. let tracked = run_pass(tracked, &active, &peer_links, &mock); assert_eq!(tracked, HashSet::from([group])); assert_eq!(mock.writes(), vec![Vec::new(), vec![member("rear0", 0)]]); @@ -957,7 +1019,7 @@ mod tests { /// in the imported set or any trigger, so only the startup seed can /// re-initialize it. #[test] - fn startup_seeds_tracked_from_dpd_and_drains_orphans() { + fn startup_seed_drains_orphans() { let group = underlay(9); let mock = MockDpd::new( found(vec![member("rear0", 0)]), @@ -970,6 +1032,7 @@ mod tests { .unwrap(); let seeded: HashSet = rt .block_on(mock.member_group_ips(&log)) + .unwrap() .into_iter() .collect(); assert!(seeded.contains(&group)); @@ -1005,12 +1068,11 @@ mod tests { } #[test] - fn withdrawn_group_with_read_failure_stays_tracked() { + fn read_retry_keeps_withdrawn_group() { let group = underlay(1); let imported = HashSet::new(); let peer_links = HashMap::new(); - let mock = - MockDpd::new(FetchOutcome::ReadFailed, WriteOutcome::Updated); + let mock = MockDpd::new(FetchOutcome::Retry, WriteOutcome::Updated); // A withdrawn group must not drop out on a transient read failure, or // its stale replication would stay programmed until a later re-import. @@ -1018,19 +1080,6 @@ mod tests { assert!(mock.writes().is_empty()); } - #[test] - fn withdrawn_group_with_read_timeout_stays_tracked() { - let group = underlay(1); - let imported = HashSet::new(); - let peer_links = HashMap::new(); - let mock = MockDpd::new(FetchOutcome::TimedOut, WriteOutcome::Updated); - - // A read stall is treated like any other transient read failure: the - // withdrawn group stays tracked so a later pass can drain it. - assert!(reconcile(group, &imported, &peer_links, &mock)); - assert!(mock.writes().is_empty()); - } - #[test] fn withdrawn_group_with_no_members_drops_without_writing() { let group = underlay(1); @@ -1043,7 +1092,7 @@ mod tests { } #[test] - fn withdrawn_group_with_members_is_emptied_then_drops() { + fn withdrawn_group_drains_then_drops() { let group = underlay(1); let imported = HashSet::new(); let peer_links = HashMap::new(); @@ -1057,48 +1106,27 @@ mod tests { } #[test] - fn withdrawn_group_with_empty_write_failure_stays_tracked() { + fn empty_write_retry_keeps_withdrawn_group() { let group = underlay(1); let imported = HashSet::new(); let peer_links = HashMap::new(); let mock = - MockDpd::new(found(vec![member("rear0", 0)]), WriteOutcome::Failed); - - assert!(reconcile(group, &imported, &peer_links, &mock)); - assert_eq!(mock.writes(), vec![Vec::::new()]); - } + MockDpd::new(found(vec![member("rear0", 0)]), WriteOutcome::Retry); - #[test] - fn withdrawn_group_with_empty_write_timeout_stays_tracked() { - let group = underlay(1); - let imported = HashSet::new(); - let peer_links = HashMap::new(); - let mock = MockDpd::new( - found(vec![member("rear0", 0)]), - WriteOutcome::TimedOut, - ); - - // The empty write stalled, so the group stays tracked to retry the - // drain on the next pass. assert!(reconcile(group, &imported, &peer_links, &mock)); assert_eq!(mock.writes(), vec![Vec::::new()]); } #[test] - fn withdrawn_group_with_tag_reassigned_on_empty_drops() { + fn stale_empty_write_drops_withdrawn_group() { let group = underlay(1); let imported = HashSet::new(); let peer_links = HashMap::new(); - let mock = MockDpd::new( - found(vec![member("rear0", 0)]), - WriteOutcome::TagReassigned, - ); + let mock = + MockDpd::new(found(vec![member("rear0", 0)]), WriteOutcome::Stale); - // The group was reassigned under a tag we no longer hold, so `ddmd` - // abandons it rather than retrying. - // - // This is distinct from the active-group case, where a reassigned tag - // stays tracked for a fresh read. + // The group disappeared or was reassigned after the read, so `ddmd` + // abandons the withdrawn group rather than retrying. assert!(!reconcile(group, &imported, &peer_links, &mock)); assert_eq!(mock.writes(), vec![Vec::::new()]); } @@ -1131,7 +1159,7 @@ mod tests { } #[test] - fn active_group_with_unresolved_nexthop_preserves_existing_member() { + fn unresolved_active_group_preserves_members() { let peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); let group = underlay(1); let imported = HashSet::from([route(peer, group)]); @@ -1149,41 +1177,68 @@ mod tests { assert!(mock.writes().is_empty()); } + /// Encodes the fail-open merge policy: while any next hop is unresolved, + /// a stale DPD member that no import accounts for, even one unrelated to + /// the unresolved next hop, persists rather than being dropped. #[test] - fn active_group_with_tag_reassigned_stays_tracked() { + fn unresolved_nexthop_retains_unrelated_stale_member() { + let resolved = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let unresolved_peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 2); + let group = underlay(1); + let imported = HashSet::from([ + route(resolved, group), + route(unresolved_peer, group), + ]); + let peer_links = HashMap::from([(resolved, rear("rear0", 0))]); + + // DPD holds the resolved member plus a stale one ("rear1") that no + // current import accounts for. + let mock = MockDpd::new( + found(vec![member("rear0", 0), member("rear1", 0)]), + WriteOutcome::Updated, + ); + + // The stale member cannot be distinguished from one owned by the + // unresolved next hop, so the merge preserves it and no write occurs. + assert!(reconcile(group, &imported, &peer_links, &mock)); + assert!(mock.writes().is_empty()); + } + + #[test] + fn stale_write_keeps_active_group() { let peer = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); let group = underlay(1); let imported = HashSet::from([route(peer, group)]); let peer_links = HashMap::from([(peer, rear("rear0", 0))]); - let mock = MockDpd::new(found(Vec::new()), WriteOutcome::TagReassigned); + let mock = MockDpd::new(found(Vec::new()), WriteOutcome::Stale); - // The write was rejected because the tag changed, but the group is - // still active, so it stays tracked to retry with a fresh read next - // pass. + // The group changed after the read, but remains imported, so it stays + // tracked to retry with a fresh read next pass. assert!(reconcile(group, &imported, &peer_links, &mock)); assert_eq!(mock.writes(), vec![vec![member("rear0", 0)]]); } #[test] - fn notify_collapses_routes_to_one_trigger_per_group() { + fn notify_deduplicates_groups() { let group_a = underlay(1); let group_b = underlay(2); let peer_a = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); let peer_b = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 2); - // Two next hops on group_a and one on group_b. The sweep should wake - // once per distinct group, not once per route. let routes = [ route(peer_a, group_a), route(peer_b, group_a), route(peer_a, group_b), ]; - let (tx, rx) = std::sync::mpsc::channel(); + let (tx, mut rx) = + tokio::sync::mpsc::channel(crate::MCAST_NOTIFY_CHANNEL_DEPTH); notify_affected_groups(routes.iter(), &tx); - drop(tx); - let signalled: Vec = rx.into_iter().collect(); + let mut signalled = Vec::new(); + while let Ok(group) = rx.try_recv() { + signalled.push(group); + } assert_eq!(signalled.len(), 2); assert_eq!( signalled.into_iter().collect::>(), diff --git a/ddm/src/sm/mod.rs b/ddm/src/sm/mod.rs index eb01cb4f8..160cabd0b 100644 --- a/ddm/src/sm/mod.rs +++ b/ddm/src/sm/mod.rs @@ -4,8 +4,8 @@ //! State machine type definitions and the [`StateMachine`] handle. The //! routing state machine implementation (discovery, solicit, exchange) lives -//! in the [`state`] submodule and is illumos-only, since it programs kernel -//! routes via [`crate::sys`] and reads interface addressing through `libnet`. +//! in the `state` submodule and is illumos-only, since it programs kernel +//! routes via `crate::sys` and reads interface addressing through `libnet`. use crate::db::Db; use crate::discovery::{self, Version}; @@ -33,6 +33,17 @@ pub enum AdminEvent { /// Withdraw a set of IPv6 prefixes Withdraw(PrefixSet), + /// Announce a set of multicast origins to peers. + AnnounceMulticast(HashSet), + + /// Withdraw a set of multicast origins. Each state machine revalidates + /// remaining reachability against the database when it processes the + /// event, rather than acting on a snapshot captured at request time. + /// The modification lands before this event is enqueued, so the processing- + /// time read is guaranteed to observe the withdrawal, and it also + /// observes any later import that has since restored reachability. + WithdrawMulticast(HashSet), + /// Expire the peer at the specified address Expire(Ipv6Addr), @@ -44,7 +55,6 @@ pub enum AdminEvent { pub enum PrefixSet { Underlay(HashSet), Tunnel(HashSet), - Multicast(HashSet), } #[derive(Debug)] @@ -188,6 +198,11 @@ pub struct InterfaceState { pub fsm_state: Mutex, pub last_fsm_state_change: Mutex, pub peer_identity: Mutex>, + /// Orders route application with expiry and renumber cleanup for this + /// interface. Discovery must not take this lock: route programming can + /// perform network and kernel I/O, while discovery needs to keep updating + /// peer liveness independently. + pub route_update: Mutex<()>, } impl InterfaceState { @@ -219,6 +234,7 @@ impl Default for InterfaceState { fsm_state: Mutex::new(FsmState::Init), last_fsm_state_change: Mutex::new(Instant::now()), peer_identity: Mutex::new(None), + route_update: Mutex::new(()), } } } @@ -252,11 +268,11 @@ pub struct SmContext { pub hostname: String, pub iface: Arc, pub stats: Arc, - /// Notifies the [`crate::mcast`] sweep that an underlay group's imported + /// Notifies the `crate::mcast` sweep that an underlay group's imported /// membership changed, by sending the group's address. The sweep wakes early /// to reconcile the group's DPD members, so the control plane never touches /// DPD directly. - pub mcast_notify: Sender, + pub mcast_notify: tokio::sync::mpsc::Sender, pub log: Logger, } diff --git a/ddm/src/sm/state.rs b/ddm/src/sm/state.rs index ddfe72c0f..8c379c9ed 100644 --- a/ddm/src/sm/state.rs +++ b/ddm/src/sm/state.rs @@ -15,20 +15,32 @@ use crate::{dbg, discovery, err, exchange, inf, wrn}; use ddm_api_types::db::RouterKind; use ddm_api_types::net::TunnelOrigin; use ddm_protocol::v3::{PathVector, TunnelUpdate, UnderlayUpdate}; -use ddm_protocol::v4::{MulticastPathHop, MulticastUpdate, Update}; +use ddm_protocol::v4::{MulticastPathHop, MulticastPathVector, Update}; use libnet::get_ipaddr_info; use slog::Logger; use std::collections::HashSet; use std::net::IpAddr; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::mpsc::Receiver; +use std::sync::mpsc::{Receiver, RecvTimeoutError}; use std::thread::{sleep, spawn}; -use std::time::Duration; +use std::time::{Duration, Instant}; use crate::discovery::Version; use std::net::Ipv6Addr; +/// Cadence for the periodic pull in the [`Exchange`] state. The initial pull +/// is one-shot, so a neighbor that originates routes after we pull it, late +/// multicast group memberships for instance, would otherwise never be +/// imported absent a push from that neighbor. Pulling on this cadence +/// repairs that drift without operator intervention. It matches the +/// multicast sweep reconcile interval so both repair loops converge on the +/// same cadence. Pre-V4 peers are skipped, since their responses carry no +/// multicast half. The exchange loop checks a fixed deadline before +/// receiving another event, so a busy event queue cannot postpone the pull +/// indefinitely. +const EXCHANGE_RESYNC_INTERVAL: Duration = crate::mcast::RECONCILE_INTERVAL; + impl StateMachine { pub fn run(&mut self) -> Result<(), SmError> { let ctx = self.ctx.clone(); @@ -168,6 +180,7 @@ impl State for Solicit { self.ctx.config.if_name, "transition solicit -> exchange" ); + // The peer is now established on this link, so wake the // multicast sweep for any of its groups whose import raced // ahead of resolution. @@ -257,6 +270,7 @@ impl Exchange { version, rt.clone(), log.clone(), + exchange::UpdateMode::Redistribute, ) { sleep(Duration::from_millis(interval)); wrn!(log, if_name, "exchange pull: {e}"); @@ -267,6 +281,30 @@ impl Exchange { }); } + fn periodic_pull(&self) { + // The resync exists to repair multicast drift, and multicast first + // appears on the wire in V4. An earlier peer's response has no + // multicast half, so a pull would only replay the full underlay and + // tunnel tables while holding the route_update lock. + if self.version < Version::V4 { + return; + } + if let Err(e) = crate::exchange::pull( + self.ctx.clone(), + self.peer, + self.version, + self.ctx.rt.clone(), + self.log.clone(), + exchange::UpdateMode::ImportOnly, + ) { + wrn!( + self.log, + self.ctx.config.if_name, + "periodic exchange pull: {e}", + ); + } + } + fn wait_for_exchange_server_to_start(&self) { inf!( self.log, @@ -300,16 +338,35 @@ impl Exchange { fn expire_peer( &mut self, - exchange_thread: &tokio::task::JoinHandle<()>, + exchange_handle: &exchange::ExchangeHandle, pull_stop: &AtomicBool, ) { - exchange_thread.abort(); + exchange_handle.abort(); self.ctx.iface.clear_peer(); + self.withdraw_peer_routes(self.peer); + pull_stop.store(true, Ordering::Relaxed); + } + + /// Remove all routes imported via `peer`, clean up the forwarding state + /// derived from them, and, on transit routers, propagate withdraws to the + /// remaining peers. + /// + /// Called on peer expiry and on renumber, where a re-advertisement + /// carries a new peer address and `peer` is the prior one. + fn withdraw_peer_routes(&self, peer: Ipv6Addr) { + // Exchange updates take the same lock. If an old-peer update is + // already running, cleanup follows it and removes its imports. If + // cleanup wins, the update subsequently observes the changed peer + // identity and is discarded. + let _route_update = mg_common::lock!(self.ctx.iface.route_update); + let removed = self.ctx.db.remove_nexthop_routes(peer); + self.redistribute_removed_routes(&removed); let crate::db::RemovedNexthopRoutes { underlay: to_remove, tunnel: to_remove_tnl, multicast: to_remove_mcast, - } = self.ctx.db.remove_nexthop_routes(self.peer); + .. + } = removed; let mut routes: Vec = Vec::new(); for x in &to_remove { let mut r: crate::sys::Route = x.clone().into(); @@ -336,27 +393,34 @@ impl Exchange { ); } - // The expired peer is gone from the imported set, so notify the + // The peer's routes are gone from the imported set, so we notify the // multicast sweep of each affected underlay group. The sweep drops the // peer's replication membership from DPD. crate::mcast::notify_affected_groups( to_remove_mcast.iter(), &self.ctx.mcast_notify, ); + } - // if we're a transit router propagate withdraws for the - // expired peer. + fn redistribute_removed_routes( + &self, + removed: &crate::db::RemovedNexthopRoutes, + ) { + // If we're a transit router propagate withdraws for the + // removed routes. if self.ctx.config.kind == RouterKind::Transit { dbg!( self.log, self.ctx.config.if_name, - "redistributing expire to {} peers", + "redistributing withdraws to {} peers", self.ctx.event_channels.len() ); - let underlay = crate::non_empty(to_remove).map(|set| { + let underlay = (!removed.underlay.is_empty()).then(|| { UnderlayUpdate::withdraw( - set.iter() + removed + .underlay + .iter() .map(|x| PathVector { destination: x.destination, path: { @@ -369,29 +433,40 @@ impl Exchange { ) }); - let tunnel = crate::non_empty(to_remove_tnl).map(|set| { + let tunnel = (!removed.tunnel.is_empty()).then(|| { TunnelUpdate::withdraw( - set.iter().cloned().map(Into::into).collect(), + removed.tunnel.iter().cloned().map(Into::into).collect(), ) }); - // Build multicast withdrawal with our hop info. - let multicast = crate::non_empty(to_remove_mcast).map(|set| { - let hop = MulticastPathHop::new( - self.ctx.hostname.clone(), - self.ctx.config.addr, + // Downstream peers collapse all paths through us into one route. + // For each removed origin, either withdraw the final path or + // refresh the peer with a remaining imported/local path. + let multicast = if removed.multicast.is_empty() { + None + } else { + let withdrawals: HashSet<_> = removed + .multicast + .iter() + .map(|route| MulticastPathVector { + origin: (&route.origin).into(), + path: Vec::new(), + }) + .collect(); + let update = crate::exchange::reconcile_multicast_withdrawals( + &withdrawals, + &removed.mcast_reachability, + &MulticastPathHop::new( + self.ctx.hostname.clone(), + self.ctx.config.addr, + ), ); - MulticastUpdate::withdraw( - set.iter() - .map(|route| { - ddm_api_types::exchange::MulticastPathVector { - origin: (&route.origin).into(), - path: vec![hop.clone()], - } - }) - .collect(), - ) - }); + if update.announce.is_empty() && update.withdraw.is_empty() { + None + } else { + Some(update) + } + }; let push = Arc::new(Update { underlay, @@ -399,11 +474,17 @@ impl Exchange { multicast, }); for ec in &self.ctx.event_channels { - ec.send(Event::Peer(PeerEvent::Push(Arc::clone(&push)))) - .unwrap(); + if let Err(e) = + ec.send(Event::Peer(PeerEvent::Push(Arc::clone(&push)))) + { + err!( + self.log, + self.ctx.config.if_name, + "deliver redistributed withdraw: {e}", + ); + } } } - pull_stop.store(true, Ordering::Relaxed); } } @@ -413,7 +494,7 @@ impl State for Exchange { event: Receiver, ) -> (Box, Receiver) { self.ctx.iface.transition(FsmState::Exchange); - let exchange_thread = loop { + let exchange_handle = loop { match exchange::handler( self.ctx.clone(), self.ctx.config.addr, @@ -443,9 +524,18 @@ impl State for Exchange { // loop below. self.initial_pull(pull_stop.clone()); + let mut resync_deadline = Instant::now() + EXCHANGE_RESYNC_INTERVAL; loop { - let e = match event.recv() { + if Instant::now() >= resync_deadline { + self.periodic_pull(); + resync_deadline = Instant::now() + EXCHANGE_RESYNC_INTERVAL; + } + + let wait = + resync_deadline.saturating_duration_since(Instant::now()); + let e = match event.recv_timeout(wait) { Ok(e) => e, + Err(RecvTimeoutError::Timeout) => continue, Err(e) => { err!( self.log, @@ -486,7 +576,7 @@ impl State for Exchange { "expiring peer {} due to failed announce", self.peer, ); - self.expire_peer(&exchange_thread, &pull_stop); + self.expire_peer(&exchange_handle, &pull_stop); return ( Box::new(Solicit::new( self.ctx.clone(), @@ -520,7 +610,7 @@ impl State for Exchange { "expiring peer {} due to failed tunnel announce", self.peer, ); - self.expire_peer(&exchange_thread, &pull_stop); + self.expire_peer(&exchange_handle, &pull_stop); return ( Box::new(Solicit::new( self.ctx.clone(), @@ -560,7 +650,7 @@ impl State for Exchange { "expiring peer {} due to failed withdraw", self.peer, ); - self.expire_peer(&exchange_thread, &pull_stop); + self.expire_peer(&exchange_handle, &pull_stop); return ( Box::new(Solicit::new( self.ctx.clone(), @@ -594,7 +684,7 @@ impl State for Exchange { "expiring peer {} due to failed tunnel withdraw", self.peer, ); - self.expire_peer(&exchange_thread, &pull_stop); + self.expire_peer(&exchange_handle, &pull_stop); return ( Box::new(Solicit::new( self.ctx.clone(), @@ -604,9 +694,7 @@ impl State for Exchange { ); } } - Event::Admin(AdminEvent::Announce(PrefixSet::Multicast( - groups, - ))) => { + Event::Admin(AdminEvent::AnnounceMulticast(groups)) => { // Build a `MulticastPathVector` for each origin, recording // our hop in the path. let hop = MulticastPathHop::new( @@ -643,7 +731,7 @@ impl State for Exchange { "expiring peer {} due to failed multicast announce", self.peer, ); - self.expire_peer(&exchange_thread, &pull_stop); + self.expire_peer(&exchange_handle, &pull_stop); return ( Box::new(Solicit::new( self.ctx.clone(), @@ -653,34 +741,77 @@ impl State for Exchange { ); } } - Event::Admin(AdminEvent::Withdraw(PrefixSet::Multicast( - groups, - ))) => { - // Build a `MulticastPathVector` for each origin, recording - // our hop in the path. + Event::Admin(AdminEvent::WithdrawMulticast(origins)) => { + // The persistent local origins were removed by the + // modification that preceded this event. The reachability + // snapshot is read here, at processing time, so an + // import that raced the admin request is observed and + // produces a replacement announcement rather than a + // stale final withdrawal. Whenever an imported path to + // an origin remains, replace the local announcement with + // that path. Otherwise, propagate the final withdrawal. let hop = MulticastPathHop::new( self.ctx.hostname.clone(), self.ctx.config.addr, ); - let path_vectors: HashSet<_> = groups + let withdrawals: HashSet<_> = origins .iter() - .map(|origin| { - ddm_api_types::exchange::MulticastPathVector { - origin: origin.into(), - path: vec![hop.clone()], - } + .map(|origin| MulticastPathVector { + origin: origin.into(), + path: Vec::new(), }) .collect(); + let reachability = self.ctx.db.multicast_reachability(); + let update = + crate::exchange::reconcile_multicast_withdrawals( + &withdrawals, + &reachability, + &hop, + ); - if let Err(e) = crate::exchange::withdraw_multicast( - &self.ctx, - self.ctx.config.clone(), - path_vectors, - self.peer, - self.version, - self.ctx.rt.clone(), - self.log.clone(), - ) { + if !update.announce.is_empty() + && let Err(e) = crate::exchange::announce_multicast( + &self.ctx, + self.ctx.config.clone(), + update.announce, + self.peer, + self.version, + self.ctx.rt.clone(), + self.log.clone(), + ) + { + err!( + self.log, + self.ctx.config.if_name, + "replace withdrawn multicast path: {e}", + ); + wrn!( + self.log, + self.ctx.config.if_name, + "expiring peer {} due to failed multicast replacement", + self.peer, + ); + self.expire_peer(&exchange_handle, &pull_stop); + return ( + Box::new(Solicit::new( + self.ctx.clone(), + self.log.clone(), + )), + event, + ); + } + + if !update.withdraw.is_empty() + && let Err(e) = crate::exchange::withdraw_multicast( + &self.ctx, + self.ctx.config.clone(), + update.withdraw, + self.peer, + self.version, + self.ctx.rt.clone(), + self.log.clone(), + ) + { err!( self.log, self.ctx.config.if_name, @@ -692,7 +823,7 @@ impl State for Exchange { "expiring peer {} due to failed multicast withdraw", self.peer, ); - self.expire_peer(&exchange_thread, &pull_stop); + self.expire_peer(&exchange_handle, &pull_stop); return ( Box::new(Solicit::new( self.ctx.clone(), @@ -710,7 +841,7 @@ impl State for Exchange { "administratively expiring peer {}", peer, ); - self.expire_peer(&exchange_thread, &pull_stop); + self.expire_peer(&exchange_handle, &pull_stop); return ( Box::new(Solicit::new( self.ctx.clone(), @@ -727,6 +858,7 @@ impl State for Exchange { self.version, self.ctx.rt.clone(), self.log.clone(), + exchange::UpdateMode::Redistribute, ) { err!( self.log, @@ -768,7 +900,7 @@ impl State for Exchange { "expiring peer {} due to failed announce", self.peer, ); - self.expire_peer(&exchange_thread, &pull_stop); + self.expire_peer(&exchange_handle, &pull_stop); return ( Box::new(Solicit::new( self.ctx.clone(), @@ -800,7 +932,7 @@ impl State for Exchange { "expiring peer {} due to failed withdraw", self.peer, ); - self.expire_peer(&exchange_thread, &pull_stop); + self.expire_peer(&exchange_handle, &pull_stop); return ( Box::new(Solicit::new( self.ctx.clone(), @@ -834,7 +966,7 @@ impl State for Exchange { "expiring peer {} due to failed multicast announce", self.peer, ); - self.expire_peer(&exchange_thread, &pull_stop); + self.expire_peer(&exchange_handle, &pull_stop); return ( Box::new(Solicit::new( self.ctx.clone(), @@ -866,7 +998,7 @@ impl State for Exchange { "expiring peer {} due to failed multicast withdraw", self.peer, ); - self.expire_peer(&exchange_thread, &pull_stop); + self.expire_peer(&exchange_handle, &pull_stop); return ( Box::new(Solicit::new( self.ctx.clone(), @@ -884,7 +1016,7 @@ impl State for Exchange { "expiring peer {} due to discovery event", self.peer, ); - self.expire_peer(&exchange_thread, &pull_stop); + self.expire_peer(&exchange_handle, &pull_stop); return ( Box::new(Solicit::new( self.ctx.clone(), @@ -900,19 +1032,36 @@ impl State for Exchange { "expiring peer {} due to failed solicit", self.peer, ); - self.expire_peer(&exchange_thread, &pull_stop); + self.expire_peer(&exchange_handle, &pull_stop); return ( Box::new(Init::new(self.ctx.clone(), self.log.clone())), event, ); } Event::Neighbor(NeighborEvent::Advertise((addr, version))) => { + if addr != self.peer { + // A re-advertisement carrying a new address renumbers + // the peer. Expiry removes routes keyed on the + // current address only, so routes under the prior + // address would otherwise persist indefinitely. + // Withdraw them as if that address expired. + inf!( + self.log, + self.ctx.config.if_name, + "peer renumbered from {} to {addr}", + self.peer, + ); + // Rebind the running exchange handler so that + // pushes arriving under the new address cannot + // recreate routes keyed by the prior peer after + // cleanup. + exchange_handle.renumber_peer(addr); + self.withdraw_peer_routes(self.peer); + } self.peer = addr; self.version = version; - // Re-advertisement may carry a new address, so wake the - // multicast sweep for the peer's groups under it. Routes - // still keyed on the prior address are not withdrawn here. - // Peer expiry and the periodic sweep reconcile those. + // Wake the multicast sweep for the peer's groups under + // the advertised address. crate::mcast::notify_peer_groups( &self.ctx.db, addr, diff --git a/ddmd/src/main.rs b/ddmd/src/main.rs index 0c4a11bb6..e957ef1ef 100644 --- a/ddmd/src/main.rs +++ b/ddmd/src/main.rs @@ -164,7 +164,8 @@ async fn run() { // context holds the sender and signals a group's address when its imported // membership changes. The sweep started by start_mcast_sweep owns the // receiver and wakes early to reconcile the full tracked set. - let (notify_tx, notify_rx) = std::sync::mpsc::channel::(); + let (notify_tx, notify_rx) = + tokio::sync::mpsc::channel::(ddm::MCAST_NOTIFY_CHANNEL_DEPTH); let (sms, event_channels) = start_state_machines(&arg, &db, &dpd, &hostname, &rt, ¬ify_tx, &log); @@ -252,7 +253,7 @@ fn start_state_machines( dpd: &Option, hostname: &str, rt: &Arc, - notify_tx: &std::sync::mpsc::Sender, + notify_tx: &tokio::sync::mpsc::Sender, log: &Logger, ) -> ( Vec, @@ -330,7 +331,7 @@ fn start_state_machines( _dpd: &Option, _hostname: &str, _rt: &Arc, - _notify_tx: &std::sync::mpsc::Sender, + _notify_tx: &tokio::sync::mpsc::Sender, _log: &Logger, ) -> ( Vec, @@ -341,15 +342,16 @@ fn start_state_machines( /// Spawn the underlay multicast membership sweep, the multicast analog of the /// unicast import-to-DPD path `ddmd` already performs in-process. The sweep -/// runs on a dedicated thread, reconciling every tracked group on each pass, -/// woken early by a trigger and otherwise self-ticking on a periodic backstop. +/// runs as a task on the daemon's runtime, reconciling every tracked group on +/// each pass, woken early by a trigger and otherwise self-ticking on a fixed +/// interval. /// /// Takes `notify_rx` by value so any path that does not start the sweep drops /// it, closing the channel and making the state machines' notify sends fail /// fast rather than accumulate as unread. #[cfg(all(feature = "backend", target_os = "illumos"))] fn start_mcast_sweep( - notify_rx: std::sync::mpsc::Receiver, + notify_rx: tokio::sync::mpsc::Receiver, dpd: Option, db: Db, peers: Vec, @@ -360,20 +362,20 @@ fn start_mcast_sweep( // No backend: returning drops notify_rx and closes the channel. return; }; - let task_log = log.clone(); - if let Err(e) = std::thread::Builder::new() - .name("ddm-mcast-members".into()) - .spawn(move || ddm::mcast::run(db, peers, dpd, rt, notify_rx, task_log)) - { - error!(log, "failed to spawn multicast membership sweep: {e}"); + // Under --api-only there are no state machines, so no peers or imports + // exist to derive membership from. A sweep would compute an empty + // desired set for every seeded group and drain its DPD members. + if peers.is_empty() { + return; } + rt.spawn(ddm::mcast::run(db, peers, dpd, notify_rx, log)); } /// Non-illumos variant: underlay multicast replicates only on a switch, so the /// sweep never starts. Consuming `notify_rx` drops it, closing the channel. #[cfg(not(all(feature = "backend", target_os = "illumos")))] fn start_mcast_sweep( - _notify_rx: std::sync::mpsc::Receiver, + _notify_rx: tokio::sync::mpsc::Receiver, _dpd: Option, _db: Db, _peers: Vec, diff --git a/mg-common/Cargo.toml b/mg-common/Cargo.toml index 262fb77a8..501e22f4e 100644 --- a/mg-common/Cargo.toml +++ b/mg-common/Cargo.toml @@ -21,7 +21,6 @@ backoff.workspace = true smf.workspace = true uuid.workspace = true libc.workspace = true -omicron-common.workspace = true # We need this on illumos, but must omit it on other platforms [target.'cfg(target_os = "illumos")'.dependencies.libnet] diff --git a/rdb/src/db.rs b/rdb/src/db.rs index 95d4dec9a..180059a1f 100644 --- a/rdb/src/db.rs +++ b/rdb/src/db.rs @@ -1532,19 +1532,6 @@ impl Db { }); } - // Synchronously revalidate affected (S,G) routes against the - // updated unicast RIB. The poptrie rebuild triggered above is - // async, so without this the MRIB update would depend on - // the rebuild thread completing first. The linear-scan fallback - // in rpf_table's lookup is sufficient here. - for prefix in &pcn.changed { - let event = match prefix { - IpNet::V4(p) => crate::mrib::rpf::RebuildEvent::V4(Some(*p)), - IpNet::V6(p) => crate::mrib::rpf::RebuildEvent::V6(Some(*p)), - }; - self.revalidate_mrib(Some(event)); - } - self.notify(pcn); Ok(()) } diff --git a/smf/ddm/manifest.xml b/smf/ddm/manifest.xml index 5e5da1cf2..f9c7f7425 100644 --- a/smf/ddm/manifest.xml +++ b/smf/ddm/manifest.xml @@ -19,7 +19,7 @@ - + diff --git a/tests/src/ddm.rs b/tests/src/ddm.rs index 29ebf5953..503d9a228 100644 --- a/tests/src/ddm.rs +++ b/tests/src/ddm.rs @@ -656,7 +656,10 @@ async fn run_trio_tests( // server routers learn it via DDM exchange. wait_for_eq!(multicast_originated_count(&t1).await?, 0); - let mcast_origin = MulticastOrigin { + // One origin per overlay address family: the overlay group is opaque to + // ddm (any multicast IpAddr), but both families must survive the + // advertise/exchange/withdraw round trip. + let mcast_origin_v4 = MulticastOrigin { overlay_group: "233.252.0.1".parse().unwrap(), underlay_group: UnderlayMulticastIpv6::new( "ff04::100".parse().unwrap(), @@ -666,14 +669,27 @@ async fn run_trio_tests( source: None, metric: 0, }; + let mcast_origin_v6 = MulticastOrigin { + overlay_group: "ff0e::1".parse().unwrap(), + underlay_group: UnderlayMulticastIpv6::new( + "ff04::101".parse().unwrap(), + ) + .unwrap(), + vni: Vni::try_from(77u32).unwrap(), + source: None, + metric: 0, + }; - t1.advertise_multicast_groups(&vec![mcast_origin.clone()]) - .await?; + t1.advertise_multicast_groups(&vec![ + mcast_origin_v4.clone(), + mcast_origin_v6.clone(), + ]) + .await?; - wait_for_eq!(multicast_originated_count(&t1).await?, 1); + wait_for_eq!(multicast_originated_count(&t1).await?, 2); wait_for_eq!(multicast_group_count(&t1).await?, 0); - wait_for_eq!(multicast_group_count(&s1).await?, 1); - wait_for_eq!(multicast_group_count(&s2).await?, 1); + wait_for_eq!(multicast_group_count(&s1).await?, 2); + wait_for_eq!(multicast_group_count(&s2).await?, 2); println_nopipe!("multicast group advertise passed"); @@ -683,11 +699,12 @@ async fn run_trio_tests( zs1.stop_router()?; zs1.start_router(false)?; let s1 = Client::new("http://10.0.0.1:8000", log.clone()); - wait_for_eq!(multicast_group_count(&s1).await.unwrap_or(99), 1); + wait_for_eq!(multicast_group_count(&s1).await.unwrap_or(99), 2); println_nopipe!("multicast router restart passed"); - t1.withdraw_multicast_groups(&vec![mcast_origin]).await?; + t1.withdraw_multicast_groups(&vec![mcast_origin_v4, mcast_origin_v6]) + .await?; wait_for_eq!(multicast_originated_count(&t1).await?, 0); wait_for_eq!(multicast_group_count(&t1).await?, 0); From 0969755bd19928bd7b41f044444b05a3de657ac7 Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Wed, 15 Jul 2026 00:48:00 +0000 Subject: [PATCH 14/16] [ci] helios target --- .github/buildomat/jobs/test-ddm-quartet.sh | 2 +- .github/buildomat/jobs/test-ddm-trio.sh | 2 +- tests/src/ddm.rs | 33 ++++++++++++++++++++-- 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/.github/buildomat/jobs/test-ddm-quartet.sh b/.github/buildomat/jobs/test-ddm-quartet.sh index 6a4ae1373..0f2b85e65 100755 --- a/.github/buildomat/jobs/test-ddm-quartet.sh +++ b/.github/buildomat/jobs/test-ddm-quartet.sh @@ -2,7 +2,7 @@ #: #: name = "test-ddm-quartet" #: variety = "basic" -#: target = "helios-2.0" +#: target = "helios-3.0" #: rust_toolchain = "stable" #: output_rules = [ #: "/work/*.log", diff --git a/.github/buildomat/jobs/test-ddm-trio.sh b/.github/buildomat/jobs/test-ddm-trio.sh index 487b5162f..ab668eca2 100755 --- a/.github/buildomat/jobs/test-ddm-trio.sh +++ b/.github/buildomat/jobs/test-ddm-trio.sh @@ -2,7 +2,7 @@ #: #: name = "test-ddm-trio" #: variety = "basic" -#: target = "helios-2.0" +#: target = "helios-3.0" #: rust_toolchain = "stable" #: output_rules = [ #: "/work/*.log", diff --git a/tests/src/ddm.rs b/tests/src/ddm.rs index 503d9a228..c5b13404f 100644 --- a/tests/src/ddm.rs +++ b/tests/src/ddm.rs @@ -192,6 +192,32 @@ impl<'a> RouterZone<'a> { self.zone.zexec("pkill ddmd") } + /// Wait for an SMF service in this zone to come online, failing fast if + /// it lands in maintenance. A service that silently fails here otherwise + /// surfaces much later as an opaque peering assertion failure. + fn wait_for_service_online(&self, fmri: &str) -> Result<()> { + for _ in 0..30 { + let state = self + .zone + .zexec(&format!("svcs -Ho state {fmri}")) + .unwrap_or_default(); + match state.trim() { + "online" => return Ok(()), + "maintenance" => { + return Err(anyhow!( + "{fmri} entered maintenance in zone {}", + self.zone.name, + )); + } + _ => sleep(Duration::from_secs(1)), + } + } + Err(anyhow!( + "timed out waiting for {fmri} to come online in zone {}", + self.zone.name, + )) + } + fn start_router(&self, restart_dpd: bool) -> Result<()> { let addrs = self.ifx[1..] .iter() @@ -226,9 +252,8 @@ impl<'a> RouterZone<'a> { ))?; self.zone.zexec("svcadm refresh dendrite:default")?; self.zone.zexec("svcadm enable dendrite:default")?; - // wait for dendrite to come up - println_nopipe!("wait 10s for dendrite to come up ..."); - sleep(Duration::from_secs(10)); + println_nopipe!("waiting for dendrite to come online ..."); + self.wait_for_service_online("dendrite:default")?; self.zone.zexec( "svccfg -s tfport setprop config/pkt_source = none", )?; @@ -237,6 +262,8 @@ impl<'a> RouterZone<'a> { )?; self.zone.zexec("svcadm refresh tfport:default")?; self.zone.zexec("svcadm enable tfport")?; + println_nopipe!("waiting for tfport to come online ..."); + self.wait_for_service_online("tfport:default")?; } self.zone.zexec(&format!( "{} {ddm} --kind transit --dendrite {} {} &> /opt/ddmd.log &", From e4d24324c59bc25c468733c40cf4c11323406457 Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Tue, 4 Aug 2026 17:04:56 +0000 Subject: [PATCH 15/16] [deps] bump dendrite mcast pin to 72b200e5 --- .github/buildomat/test-ddm-common.sh | 2 +- Cargo.lock | 30 ++++++++-------------------- Cargo.toml | 2 +- 3 files changed, 10 insertions(+), 24 deletions(-) diff --git a/.github/buildomat/test-ddm-common.sh b/.github/buildomat/test-ddm-common.sh index 56fc5c105..c44484502 100755 --- a/.github/buildomat/test-ddm-common.sh +++ b/.github/buildomat/test-ddm-common.sh @@ -3,7 +3,7 @@ export MAGHEMITE_VERSION=`git rev-parse HEAD` export SOFTNPU_VERSION=284c6830722548714128e63ea04bcca78ee27154 export SIDECAR_LITE_VERSION=6f3311e8acd7e7e95c167aab61188355a93afe72 -export DENDRITE_VERSION=96483d0c52f0cfc4e22e70229466798254a99b6c +export DENDRITE_VERSION=72b200e571a81d0191fc9b607592f922d929a548 function cleanup { pfexec chown -R `id -un`:`id -gn` . diff --git a/Cargo.lock b/Cargo.lock index f4a568025..7ba17d303 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -905,7 +905,7 @@ dependencies = [ [[package]] name = "common" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/dendrite?rev=96483d0c52f0cfc4e22e70229466798254a99b6c#96483d0c52f0cfc4e22e70229466798254a99b6c" +source = "git+https://github.com/oxidecomputer/dendrite?rev=72b200e571a81d0191fc9b607592f922d929a548#72b200e571a81d0191fc9b607592f922d929a548" dependencies = [ "anyhow", "chrono", @@ -1689,7 +1689,7 @@ dependencies = [ [[package]] name = "dpd-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/dendrite?rev=96483d0c52f0cfc4e22e70229466798254a99b6c#96483d0c52f0cfc4e22e70229466798254a99b6c" +source = "git+https://github.com/oxidecomputer/dendrite?rev=72b200e571a81d0191fc9b607592f922d929a548#72b200e571a81d0191fc9b607592f922d929a548" dependencies = [ "async-trait", "chrono", @@ -7352,7 +7352,7 @@ version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" dependencies = [ - "heck 0.5.0", + "heck 0.4.1", "proc-macro2", "quote", "syn 2.0.117", @@ -7588,7 +7588,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c998b0c8b921495196a48aabaf1901ff28be0760136e31604f7967b0792050e" dependencies = [ "papergrid 0.11.0", - "tabled_derive 0.7.0", + "tabled_derive", "unicode-width 0.1.14", ] @@ -7599,7 +7599,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5dc662e6da844ad6e428ad16b57967c9d33c82e16bb1c258326c0c078605dff" dependencies = [ "papergrid 0.18.0", - "tabled_derive 0.11.0", "testing_table", ] @@ -7616,19 +7615,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "tabled_derive" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ea5d1b13ca6cff1f9231ffd62f15eefd72543dab5e468735f1a456728a02846" -dependencies = [ - "heck 0.5.0", - "proc-macro-error2", - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "tabwriter" version = "1.4.1" @@ -8257,12 +8243,12 @@ dependencies = [ [[package]] name = "transceiver-controller" version = "0.1.1" -source = "git+https://github.com/oxidecomputer/transceiver-control?branch=main#e88642c75928f8760ed0f09e7593f849d9bc70ac" +source = "git+https://github.com/oxidecomputer/transceiver-control?branch=main#03a79a895871f9eca5400fac1556ed8526e031b7" dependencies = [ "anyhow", "clap", "hubpack", - "itertools 0.14.0", + "itertools 0.15.0", "nix", "schemars 0.8.22", "serde", @@ -8281,7 +8267,7 @@ dependencies = [ [[package]] name = "transceiver-decode" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/transceiver-control?branch=main#e88642c75928f8760ed0f09e7593f849d9bc70ac" +source = "git+https://github.com/oxidecomputer/transceiver-control?branch=main#03a79a895871f9eca5400fac1556ed8526e031b7" dependencies = [ "schemars 0.8.22", "serde", @@ -8293,7 +8279,7 @@ dependencies = [ [[package]] name = "transceiver-messages" version = "0.1.1" -source = "git+https://github.com/oxidecomputer/transceiver-control?branch=main#e88642c75928f8760ed0f09e7593f849d9bc70ac" +source = "git+https://github.com/oxidecomputer/transceiver-control?branch=main#03a79a895871f9eca5400fac1556ed8526e031b7" dependencies = [ "bitflags 2.12.1", "clap", diff --git a/Cargo.toml b/Cargo.toml index dd07bddf7..8abfb51e8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -160,4 +160,4 @@ rev = "aa4714289cbf28010a821c49d9b73ec94074fb91" [workspace.dependencies.dpd-client] git = "https://github.com/oxidecomputer/dendrite" -rev = "96483d0c52f0cfc4e22e70229466798254a99b6c" +rev = "72b200e571a81d0191fc9b607592f922d929a548" From f74e74694b2df8c88c862b6e32409898ad85ded5 Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Thu, 6 Aug 2026 14:51:10 +0000 Subject: [PATCH 16/16] [deps] update dendrite mcast pin to 0d48b5a3 --- .github/buildomat/test-ddm-common.sh | 2 +- Cargo.lock | 30 ++++++++++++++++++++-------- Cargo.toml | 2 +- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/.github/buildomat/test-ddm-common.sh b/.github/buildomat/test-ddm-common.sh index c44484502..96170c21e 100755 --- a/.github/buildomat/test-ddm-common.sh +++ b/.github/buildomat/test-ddm-common.sh @@ -3,7 +3,7 @@ export MAGHEMITE_VERSION=`git rev-parse HEAD` export SOFTNPU_VERSION=284c6830722548714128e63ea04bcca78ee27154 export SIDECAR_LITE_VERSION=6f3311e8acd7e7e95c167aab61188355a93afe72 -export DENDRITE_VERSION=72b200e571a81d0191fc9b607592f922d929a548 +export DENDRITE_VERSION=0d48b5a39dc49d30b228ae01f04b8cc27a161e74 function cleanup { pfexec chown -R `id -un`:`id -gn` . diff --git a/Cargo.lock b/Cargo.lock index 7ba17d303..8fc8405bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -905,7 +905,7 @@ dependencies = [ [[package]] name = "common" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/dendrite?rev=72b200e571a81d0191fc9b607592f922d929a548#72b200e571a81d0191fc9b607592f922d929a548" +source = "git+https://github.com/oxidecomputer/dendrite?rev=0d48b5a39dc49d30b228ae01f04b8cc27a161e74#0d48b5a39dc49d30b228ae01f04b8cc27a161e74" dependencies = [ "anyhow", "chrono", @@ -1689,7 +1689,7 @@ dependencies = [ [[package]] name = "dpd-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/dendrite?rev=72b200e571a81d0191fc9b607592f922d929a548#72b200e571a81d0191fc9b607592f922d929a548" +source = "git+https://github.com/oxidecomputer/dendrite?rev=0d48b5a39dc49d30b228ae01f04b8cc27a161e74#0d48b5a39dc49d30b228ae01f04b8cc27a161e74" dependencies = [ "async-trait", "chrono", @@ -7352,7 +7352,7 @@ version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2", "quote", "syn 2.0.117", @@ -7588,7 +7588,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c998b0c8b921495196a48aabaf1901ff28be0760136e31604f7967b0792050e" dependencies = [ "papergrid 0.11.0", - "tabled_derive", + "tabled_derive 0.7.0", "unicode-width 0.1.14", ] @@ -7599,6 +7599,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5dc662e6da844ad6e428ad16b57967c9d33c82e16bb1c258326c0c078605dff" dependencies = [ "papergrid 0.18.0", + "tabled_derive 0.11.0", "testing_table", ] @@ -7615,6 +7616,19 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "tabled_derive" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea5d1b13ca6cff1f9231ffd62f15eefd72543dab5e468735f1a456728a02846" +dependencies = [ + "heck 0.5.0", + "proc-macro-error2", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "tabwriter" version = "1.4.1" @@ -8243,12 +8257,12 @@ dependencies = [ [[package]] name = "transceiver-controller" version = "0.1.1" -source = "git+https://github.com/oxidecomputer/transceiver-control?branch=main#03a79a895871f9eca5400fac1556ed8526e031b7" +source = "git+https://github.com/oxidecomputer/transceiver-control?branch=main#e88642c75928f8760ed0f09e7593f849d9bc70ac" dependencies = [ "anyhow", "clap", "hubpack", - "itertools 0.15.0", + "itertools 0.14.0", "nix", "schemars 0.8.22", "serde", @@ -8267,7 +8281,7 @@ dependencies = [ [[package]] name = "transceiver-decode" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/transceiver-control?branch=main#03a79a895871f9eca5400fac1556ed8526e031b7" +source = "git+https://github.com/oxidecomputer/transceiver-control?branch=main#e88642c75928f8760ed0f09e7593f849d9bc70ac" dependencies = [ "schemars 0.8.22", "serde", @@ -8279,7 +8293,7 @@ dependencies = [ [[package]] name = "transceiver-messages" version = "0.1.1" -source = "git+https://github.com/oxidecomputer/transceiver-control?branch=main#03a79a895871f9eca5400fac1556ed8526e031b7" +source = "git+https://github.com/oxidecomputer/transceiver-control?branch=main#e88642c75928f8760ed0f09e7593f849d9bc70ac" dependencies = [ "bitflags 2.12.1", "clap", diff --git a/Cargo.toml b/Cargo.toml index 8abfb51e8..f847fce2f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -160,4 +160,4 @@ rev = "aa4714289cbf28010a821c49d9b73ec94074fb91" [workspace.dependencies.dpd-client] git = "https://github.com/oxidecomputer/dendrite" -rev = "72b200e571a81d0191fc9b607592f922d929a548" +rev = "0d48b5a39dc49d30b228ae01f04b8cc27a161e74"