diff --git a/config/Cargo.toml b/config/Cargo.toml index 4c6ace6c8e..ffe89a893a 100644 --- a/config/Cargo.toml +++ b/config/Cargo.toml @@ -5,6 +5,9 @@ license.workspace = true publish.workspace = true version.workspace = true +[features] +bolero = ["dep:bolero", "lpm/bolero"] + [dependencies] # internal common = { workspace = true } @@ -16,6 +19,7 @@ net = { workspace = true } # external arc-swap = { workspace = true } +bolero = { workspace = true, optional = true, default-features = false, features = ["alloc"] } chrono = { workspace = true, features = ["alloc", "std"] } derive_builder = { workspace = true, features = [] } ipnet = { workspace = true } diff --git a/config/src/converters/k8s/config/expose.rs b/config/src/converters/k8s/config/expose.rs index 719ecf2e91..2861b37192 100644 --- a/config/src/converters/k8s/config/expose.rs +++ b/config/src/converters/k8s/config/expose.rs @@ -491,7 +491,7 @@ mod test { "10.0.4.0/24".parse::().unwrap(), ), ]); - let expose_gen = k8s_intf::bolero::expose::LegalValueExposeGenerator::new(&subnets); + let expose_gen = k8s_intf::bolero::expose::AnyExposeGenerator::new(0, &subnets); bolero::check!() .with_generator(expose_gen) .for_each(|k8s_expose| { @@ -549,6 +549,7 @@ mod test { }) .unwrap_or(vec![]); k8s_nots.sort(); + k8s_nots.dedup(); let k8s_subnets = k8s_expose .ips .as_ref() @@ -566,6 +567,7 @@ mod test { .unwrap_or(vec![]); k8s_ips.extend(k8s_subnets); k8s_ips.sort(); + k8s_ips.dedup(); let k8s_as = k8s_expose.r#as.as_ref().map(|r#as| { let mut ret = r#as @@ -574,6 +576,7 @@ mod test { .map(|r#as| r#as.cidr.as_ref().unwrap().clone()) .collect::>(); ret.sort(); + ret.dedup(); ret }); @@ -584,6 +587,7 @@ mod test { .map(|r#as| r#as.not.as_ref().unwrap().clone()) .collect::>(); ret.sort(); + ret.dedup(); ret }); diff --git a/config/src/converters/k8s/config/peering.rs b/config/src/converters/k8s/config/peering.rs index 45ae4a8e29..dc82803ab5 100644 --- a/config/src/converters/k8s/config/peering.rs +++ b/config/src/converters/k8s/config/peering.rs @@ -91,6 +91,7 @@ mod test { use k8s_intf::bolero::peering::{ LegalValuePeeringsGenerator, LegalValuePeeringsPeeringGenerator, }; + use k8s_intf::bolero::{AddressFamily, NatFlavour}; use lpm::prefix::Prefix; use crate::converters::k8s::config::{SubnetMap, VpcSubnetMap}; @@ -98,7 +99,9 @@ mod test { #[test] fn test_vpc_manifest_conversion() { let subnets = SubnetMap::new(); // Let this be empty since we are test subnet conversion elsewhere - let generator = LegalValuePeeringsPeeringGenerator::new(&subnets); + let flavours = NatFlavour::all(); + let generator = + LegalValuePeeringsPeeringGenerator::new(&subnets, &flavours, AddressFamily::V4, 3, 0); bolero::check!() .with_generator(generator) .for_each(|peering| { @@ -169,7 +172,11 @@ mod test { ]), ), ]); - let generator = LegalValuePeeringsGenerator::new(&subnets).unwrap(); + let flavours = NatFlavour::all(); + let families = AddressFamily::all(); + let groups = vec!["gwgroup-0".to_string()]; + let generator = + LegalValuePeeringsGenerator::new(&subnets, &flavours, &families, 3, &groups).unwrap(); bolero::check!() .with_generator(generator) .for_each(|peering| { diff --git a/config/src/errors.rs b/config/src/errors.rs index bbc6509359..78a81abc05 100644 --- a/config/src/errors.rs +++ b/config/src/errors.rs @@ -46,6 +46,8 @@ pub enum ConfigError { FailureApply(String), #[error("Forbidden: {0}")] Forbidden(&'static str), + #[error("Not supported yet: {0}")] + Unsupported(&'static str), #[error("Bad VPC Id")] BadVpcId(String), #[error("Bad VTEP local address {0}: {1}")] @@ -75,8 +77,21 @@ pub enum ConfigError { #[error("Invalid ACL configuration: {0}")] InvalidAcl(String), // NAT-specific - #[error("Mismatched prefixes sizes for static NAT: {0:?} and {1:?}")] + #[error( + "Mismatched sizes for static NAT: the exposed prefixes and the range they translate to \ + must cover the same number of address-port pairs (they cover {0:?} and {1:?})" + )] MismatchedPrefixSizes(PrefixWithPortsSize, PrefixWithPortsSize), + #[error( + "Mismatched prefix lengths for port forwarding: /{private} exposed and /{public} \ + translated to; a rule maps addresses one for one, so the two must be the same length" + )] + MismatchedPrefixLengths { private: u8, public: u8 }, + #[error( + "Mismatched port range sizes for port forwarding: {private} ports exposed and {public} \ + translated to; a rule maps ports one for one, so the two must be the same size" + )] + MismatchedPortRangeSizes { private: usize, public: usize }, #[error("Peering {0} has manifests using incompatible NAT modes")] IncompatibleNatModes(String), #[error("Vpc {0} has a peering with no exposes")] diff --git a/config/src/external/overlay/validation_tests.rs b/config/src/external/overlay/validation_tests.rs index 571811ebc8..b1644b38e1 100644 --- a/config/src/external/overlay/validation_tests.rs +++ b/config/src/external/overlay/validation_tests.rs @@ -646,7 +646,13 @@ mod test { .unwrap(); let result = expose.validate(); assert!( - matches!(result, Err(ConfigError::MismatchedPrefixSizes(_, _))), + matches!( + result, + Err(ConfigError::MismatchedPrefixLengths { + private: 24, + public: 25 + }) + ), "{result:?}", ); } diff --git a/config/src/external/overlay/vpcpeering.rs b/config/src/external/overlay/vpcpeering.rs index d5f641a26c..c64919a5c7 100644 --- a/config/src/external/overlay/vpcpeering.rs +++ b/config/src/external/overlay/vpcpeering.rs @@ -432,8 +432,6 @@ impl VpcExpose { // - we have no exclusion prefixes (note: we could relax this constraint now that we // collapse exclusion prefixes early) // - we have a single prefix on each side (private and public addresses) - // - we have the same number of addresses on each side - // - the list of associated port ranges also has the same size on each side if collapsed_expose.has_port_forwarding() { if !self.nots.is_empty() || !self.not_as_or_empty().is_empty() { return Err(ConfigError::Forbidden( @@ -446,12 +444,6 @@ impl VpcExpose { "Port forwarding requires a single prefix on each side", )); } - if ips_sizes != as_range_sizes { - return Err(ConfigError::MismatchedPrefixSizes( - ips_sizes, - as_range_sizes, - )); - } // For port forwarding, ensure that a port range is always present. Lack of port range would imply // all ports, which is not allowed since port 0 is forbidden in the implementation for prefixes in [collapsed_expose.ips(), collapsed_expose.as_range_or_empty()] { @@ -461,6 +453,29 @@ impl VpcExpose { )); } } + + let internal = collapsed_expose + .ips() + .first() + .unwrap_or_else(|| unreachable!()); + let external = collapsed_expose + .as_range_or_empty() + .first() + .unwrap_or_else(|| unreachable!()); + if internal.prefix().length() != external.prefix().length() { + return Err(ConfigError::MismatchedPrefixLengths { + private: internal.prefix().length(), + public: external.prefix().length(), + }); + } + let internal_ports = internal.ports().unwrap_or_else(|| unreachable!()); + let external_ports = external.ports().unwrap_or_else(|| unreachable!()); + if internal_ports.len() != external_ports.len() { + return Err(ConfigError::MismatchedPortRangeSizes { + private: internal_ports.len(), + public: external_ports.len(), + }); + } } // For masquerade, we don't support port ranges @@ -1005,3 +1020,417 @@ impl VpcPeeringTable { .filter(move |p| p.left.name == vpc || p.right.name == vpc) } } + +#[cfg(any(test, feature = "bolero"))] +pub mod contract { + + use super::{VpcExpose, VpcExposeNatConfig, VpcManifest, VpcPeering, VpcPeeringTable}; + use crate::ConfigError; + use crate::external::overlay::vpc::{Vpc, VpcTable}; + use crate::external::overlay::{Overlay, ValidatedOverlay}; + use bolero::{Driver, ValueGenerator}; + use lpm::prefix::{ + IpPrefix, Ipv4Prefix, Ipv6Prefix, L4Protocol, PortRange, Prefix, PrefixWithOptionalPorts, + }; + use std::net::{Ipv4Addr, Ipv6Addr}; + use std::ops::Bound::Included; + use std::time::Duration; + + const MAX_HOST_BITS: u8 = 8; + + const MAX_PORTS: u16 = 1024; + + #[derive(Debug, Clone, Copy, Default)] + pub struct PortForwardingExpose; + + impl ValueGenerator for PortForwardingExpose { + type Output = VpcExpose; + + fn generate(&self, driver: &mut D) -> Option { + let host_bits = driver.gen_u8(Included(&0), Included(&MAX_HOST_BITS))?; + let (internal, external) = if driver.produce::()? { + v4_pair(driver, host_bits)? + } else { + v6_pair(driver, host_bits)? + }; + + let count = driver.gen_u16(Included(&1), Included(&MAX_PORTS))?; + let internal_ports = port_range(driver, count)?; + let external_ports = port_range(driver, count)?; + + let proto = match driver.gen_u8(Included(&0), Included(&2))? { + 0 => L4Protocol::Tcp, + 1 => L4Protocol::Udp, + _ => L4Protocol::Any, + }; + let idle_timeout = match driver.gen_u8(Included(&0), Included(&2))? { + 0 => None, + 1 => Some(Duration::from_secs(5)), + _ => Some(Duration::from_mins(5)), + }; + + VpcExpose::empty() + .make_port_forwarding(idle_timeout, Some(proto)) + .ok()? + .ip(PrefixWithOptionalPorts::new(internal, Some(internal_ports))) + .as_range(PrefixWithOptionalPorts::new(external, Some(external_ports))) + .ok() + } + } + + #[derive(Debug, Clone, Copy, Default)] + pub struct MasqueradeExpose; + + impl ValueGenerator for MasqueradeExpose { + type Output = VpcExpose; + + fn generate(&self, driver: &mut D) -> Option { + let v4 = driver.produce::()?; + let privates = driver.gen_u8(Included(&1), Included(&3))?; + let publics = driver.gen_u8(Included(&1), Included(&2))?; + let base = driver.produce::()?; + let idle_timeout = match driver.gen_u8(Included(&0), Included(&2))? { + 0 => None, + 1 => Some(Duration::from_secs(30)), + _ => Some(Duration::from_mins(2)), + }; + + let mut expose = VpcExpose::empty().make_masquerade(idle_timeout).ok()?; + for index in 0..privates { + expose = expose.ip(PrefixWithOptionalPorts::new( + block(v4, Side::Private, base.wrapping_add(index))?, + None, + )); + } + for index in 0..publics { + expose = expose + .as_range(PrefixWithOptionalPorts::new( + block(v4, Side::Public, base.wrapping_add(index))?, + None, + )) + .ok()?; + } + Some(expose) + } + } + + #[derive(Clone, Copy)] + enum Side { + Private, + Public, + } + + fn block(v4: bool, side: Side, index: u8) -> Option { + if v4 { + let bits = match side { + Side::Private => 0x0A00_0000 | (u32::from(index) << 16), + Side::Public => 0xAC10_0000 | (u32::from(index) << 8), + }; + prefix_v4(bits, 24) + } else { + let selector = match side { + Side::Private => 0u128, + Side::Public => 1, + }; + let bits = (0x2001_0db8u128 << 96) | (selector << 80) | (u128::from(index) << 64); + prefix_v6(bits, 64) + } + } + + #[derive(Debug, Clone, Copy, Default)] + pub struct StaticNatExpose; + + const MAX_TOTAL_LOG: u8 = 6; + + impl ValueGenerator for StaticNatExpose { + type Output = VpcExpose; + + fn generate(&self, driver: &mut D) -> Option { + let v4 = driver.produce::()?; + let total_log = driver.gen_u8(Included(&0), Included(&MAX_TOTAL_LOG))?; + + let privates = place(v4, Side::Private, &split(driver, total_log)?)?; + let publics = place(v4, Side::Public, &split(driver, total_log)?)?; + + let mut expose = VpcExpose::empty().make_static_nat().ok()?; + for prefix in privates { + expose = expose.ip(PrefixWithOptionalPorts::new(prefix, None)); + } + for prefix in publics { + expose = expose + .as_range(PrefixWithOptionalPorts::new(prefix, None)) + .ok()?; + } + Some(expose) + } + } + + fn split(driver: &mut D, total_log: u8) -> Option> { + let mut parts = vec![total_log]; + for _ in 0..driver.gen_u8(Included(&0), Included(&3))? { + let splittable: Vec = parts + .iter() + .enumerate() + .filter(|(_, log)| **log > 0) + .map(|(index, _)| index) + .collect(); + if splittable.is_empty() { + break; + } + let choice = usize::from(driver.gen_u8( + Included(&0), + Included(&u8::try_from(splittable.len() - 1).ok()?), + )?); + let log = parts.swap_remove(splittable[choice]); + parts.push(log - 1); + parts.push(log - 1); + } + parts.sort_unstable_by(|a, b| b.cmp(a)); + Some(parts) + } + + fn place(v4: bool, side: Side, parts: &[u8]) -> Option> { + let mut cursor = if v4 { + u128::from(match side { + Side::Private => 0x0A00_0000u32, + Side::Public => 0xAC10_0000, + }) + } else { + let selector = match side { + Side::Private => 0u128, + Side::Public => 1, + }; + (0x2001_0db8u128 << 96) | (selector << 80) + }; + + let mut out = Vec::with_capacity(parts.len()); + for &log in parts { + let prefix = if v4 { + prefix_v4(u32::try_from(cursor).ok()?, 32 - log)? + } else { + prefix_v6(cursor, 128 - log)? + }; + out.push(prefix); + cursor += 2u128 << log; + } + Some(out) + } + + pub const LOCAL_VNI: u32 = 100; + pub const REMOTE_VNI: u32 = 200; + + pub fn overlay_offering(expose: VpcExpose) -> Result { + overlay_with(expose)?.validate() + } + + pub fn overlay_with(expose: VpcExpose) -> Result { + overlay_with_exposes(vec![expose]) + } + + pub fn overlay_with_exposes(exposes: Vec) -> Result { + let remote_prefix = match exposes + .first() + .and_then(|expose| expose.ips.first().map(PrefixWithOptionalPorts::prefix)) + { + Some(Prefix::IPV6(_)) => "2001:db8:ffff::/64", + _ => "3.3.3.0/24", + }; + + let mut vpc_table = VpcTable::new(); + vpc_table.add(Vpc::new("VPC-1", "AAAAA", LOCAL_VNI)?)?; + vpc_table.add(Vpc::new("VPC-2", "BBBBB", REMOTE_VNI)?)?; + + let local = exposes + .into_iter() + .fold(VpcManifest::new("VPC-1"), VpcManifest::exposing); + let remote = VpcManifest::new("VPC-2").exposing( + VpcExpose::empty().ip(remote_prefix + .parse::() + .unwrap_or_else(|_| unreachable!()) + .into()), + ); + let mut peerings = VpcPeeringTable::new(); + peerings.add(VpcPeering::with_default_group( + "VPC-1--VPC-2", + local, + remote, + ))?; + + Ok(Overlay::new(vpc_table, peerings)) + } + + #[must_use] + pub fn nat_config(expose: &VpcExpose) -> Option<&VpcExposeNatConfig> { + expose.nat_config() + } + + fn v4_pair(driver: &mut D, host_bits: u8) -> Option<(Prefix, Prefix)> { + let len = 32 - host_bits; + let mask = u32::MAX.checked_shl(u32::from(host_bits)).unwrap_or(0); + let internal = (0x0A00_0000 | (driver.produce::()? & 0x00FF_FFFF)) & mask; + let external = (0xAC10_0000 | (driver.produce::()? & 0x000F_FFFF)) & mask; + Some((prefix_v4(internal, len)?, prefix_v4(external, len)?)) + } + + const INTERNAL_BASE: u128 = 0x2001_0db8_0000_0000_0000_0000_0000_0000; + const EXTERNAL_BASE: u128 = 0x2001_0db8_0001_0000_0000_0000_0000_0000; + + fn v6_pair(driver: &mut D, host_bits: u8) -> Option<(Prefix, Prefix)> { + let len = 128 - host_bits; + let mask = u128::MAX.checked_shl(u32::from(host_bits)).unwrap_or(0); + let internal = (INTERNAL_BASE | u128::from(driver.produce::()?)) & mask; + let external = (EXTERNAL_BASE | u128::from(driver.produce::()?)) & mask; + Some((prefix_v6(internal, len)?, prefix_v6(external, len)?)) + } + + fn prefix_v4(bits: u32, len: u8) -> Option { + Ipv4Prefix::new(Ipv4Addr::from_bits(bits), len) + .ok() + .map(Prefix::from) + } + + fn prefix_v6(bits: u128, len: u8) -> Option { + Ipv6Prefix::new(Ipv6Addr::from_bits(bits), len) + .ok() + .map(Prefix::from) + } + + fn port_range(driver: &mut D, count: u16) -> Option { + let last_start = u16::MAX - (count - 1); + let start = driver.gen_u16(Included(&1), Included(&last_start))?; + PortRange::new(start, start + (count - 1)).ok() + } + + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn every_generated_expose_validates() { + bolero::check!() + .with_generator(PortForwardingExpose) + .for_each(|expose: &VpcExpose| { + let validated = expose.validate(); + assert!( + validated.is_ok(), + "generated expose was rejected: {expose} -- {:?}", + validated.err() + ); + }); + } + + fn forwarding(internal: (&str, u16, u16), external: (&str, u16, u16)) -> VpcExpose { + let side = |(prefix, first, last): (&str, u16, u16)| { + PrefixWithOptionalPorts::new( + prefix.into(), + Some(PortRange::new(first, last).unwrap_or_else(|_| unreachable!())), + ) + }; + VpcExpose::empty() + .make_port_forwarding(None, None) + .unwrap_or_else(|_| unreachable!()) + .ip(side(internal)) + .as_range(side(external)) + .unwrap_or_else(|_| unreachable!()) + } + + #[test] + fn compensating_sizes_do_not_make_a_valid_expose() { + let expose = forwarding(("10.0.0.0/32", 1000, 1099), ("172.16.0.0/30", 2000, 2024)); + assert!( + matches!( + expose.validate(), + Err(ConfigError::MismatchedPrefixLengths { + private: 32, + public: 30 + }) + ), + "a /32 with 100 ports opposite a /30 with 25 was accepted: {:?}", + expose.validate() + ); + } + + #[test] + fn port_ranges_of_different_sizes_do_not_make_a_valid_expose() { + let expose = forwarding(("10.0.0.0/32", 1000, 1099), ("172.16.0.0/32", 2000, 2049)); + assert!( + matches!( + expose.validate(), + Err(ConfigError::MismatchedPortRangeSizes { + private: 100, + public: 50 + }) + ), + "100 ports opposite 50 was accepted: {:?}", + expose.validate() + ); + } + + #[test] + fn matched_sides_still_make_a_valid_expose() { + let expose = forwarding(("10.0.0.0/30", 1000, 1099), ("172.16.0.0/30", 2000, 2099)); + assert!(expose.validate().is_ok(), "{:?}", expose.validate()); + } + + #[test] + fn every_generated_masquerade_expose_validates() { + bolero::check!() + .with_generator(MasqueradeExpose) + .for_each(|expose: &VpcExpose| { + let validated = expose.validate().unwrap_or_else(|e| { + panic!("generated expose was rejected: {expose} -- {e:?}") + }); + assert!(validated.has_masquerade()); + assert!(!validated.ips().is_empty()); + assert!(!validated.as_range_or_empty().is_empty()); + }); + } + + #[test] + fn a_generated_expose_can_be_offered_in_an_overlay() { + bolero::check!() + .with_generator(MasqueradeExpose) + .cloned() + .for_each(|expose: VpcExpose| { + let shown = expose.to_string(); + assert!( + overlay_offering(expose).is_ok(), + "could not build an overlay around {shown}" + ); + }); + } + + #[test] + fn every_generated_static_nat_expose_validates() { + let mut shapes_differed = false; + bolero::check!() + .with_generator(StaticNatExpose) + .for_each(|expose: &VpcExpose| { + let validated = expose.validate().unwrap_or_else(|e| { + panic!("generated expose was rejected: {expose} -- {e:?}") + }); + assert!(validated.has_static_nat()); + if validated.ips().len() != validated.as_range_or_empty().len() { + shapes_differed = true; + } + }); + assert!( + shapes_differed, + "no generated expose had a different number of prefixes on each side, so the \ + mapping was never asked to fragment" + ); + } + + #[test] + fn a_generated_expose_survives_validation_as_port_forwarding() { + bolero::check!() + .with_generator(PortForwardingExpose) + .for_each(|expose: &VpcExpose| { + let validated = expose.validate().unwrap_or_else(|e| panic!("{e:?}")); + assert!(validated.has_port_forwarding()); + assert_eq!(validated.ips().len(), 1); + assert_eq!(validated.as_range_or_empty().len(), 1); + }); + } + } +} diff --git a/flow-entry/src/flow_table/mod.rs b/flow-entry/src/flow_table/mod.rs index a8d246a881..10307d6ace 100644 --- a/flow-entry/src/flow_table/mod.rs +++ b/flow-entry/src/flow_table/mod.rs @@ -9,7 +9,7 @@ pub mod table; mod concurrent_fuzz; pub use nf_lookup::FlowLookup; -pub use table::{FlowTable, FlowTableReadGuard}; +pub use table::{FlowTable, FlowTableReadGuard, Insertion}; pub use net::flows::atomic_instant::AtomicInstant; pub use net::flows::flow_info::*; diff --git a/flow-entry/src/flow_table/table.rs b/flow-entry/src/flow_table/table.rs index 27695322f7..9972072937 100644 --- a/flow-entry/src/flow_table/table.rs +++ b/flow-entry/src/flow_table/table.rs @@ -19,6 +19,12 @@ pub enum FlowTableError { CapacityExceeded, } +#[derive(Debug)] +pub enum Insertion { + Installed, + Occupied(Arc), +} + type Table = DashMap, RandomState>; #[derive(Debug)] @@ -212,25 +218,41 @@ impl FlowTable { }); } + fn admit(&self, table: &Table, val: &Arc) -> Result<(), FlowTableError> { + if table.len() < self.capacity.load(Ordering::Relaxed) { + return Ok(()); + } + let has_related_in_table = val + .related + .as_ref() + .and_then(Weak::upgrade) + .is_some_and(|rel| rel.is_active()); + + if has_related_in_table { + Ok(()) + } else { + Err(FlowTableError::CapacityExceeded) + } + } + + fn displace(old: Option<&Arc>) { + let Some(old) = old else { + return; + }; + old.update_status(FlowStatus::Detached); + old.token.cancel(); + if let Some(related) = old.related.as_ref().and_then(Weak::upgrade) { + debug!("insert: invalidating the partner of a displaced flow"); + related.invalidate(); + } + } + fn insert_common(&self, val: &Arc) -> Result>, FlowTableError> { let table = self.table.read(); - let capacity = self.capacity.load(Ordering::Relaxed); let flow_key = val.flowkey(); debug!("insert: inserting flow {flow_key}"); - // Reject new flows when at capacity. Exception: always admit the second half of a - // related pair (e.g. the reverse NAT flow) to avoid leaving a one-sided entry. - if table.len() >= capacity { - let has_related_in_table = val - .related - .as_ref() - .and_then(Weak::upgrade) - .is_some_and(|rel| rel.is_active()); - - if !has_related_in_table { - return Err(FlowTableError::CapacityExceeded); - } - } + self.admit(&table, val)?; let result = table.insert(*flow_key, val.clone()); // Set Active only after the insert so that the invariant holds: Active iff in the @@ -243,10 +265,7 @@ impl FlowTable { #[cfg(not(any(feature = "shuttle", feature = "loom")))] Self::start_timer(self.table.clone(), val.clone()); - if let Some(old) = result.as_ref() { - old.update_status(FlowStatus::Detached); - old.token.cancel(); - } + Self::displace(result.as_ref()); let Some(ret) = result else { return Ok(None); @@ -259,6 +278,46 @@ impl FlowTable { Ok(Some(ret)) } + pub fn insert_if_absent(&self, val: &Arc) -> Result { + let table = self.table.read(); + let flow_key = val.flowkey(); + debug!("insert: inserting flow {flow_key} unless it is already held"); + + self.admit(&table, val)?; + + let displaced = match table.entry(*flow_key) { + dashmap::Entry::Occupied(mut occupied) => { + if occupied.get().is_active() { + Err(occupied.get().clone()) + } else { + Ok(Some(occupied.insert(val.clone()))) + } + } + dashmap::Entry::Vacant(vacant) => { + vacant.insert(val.clone()); + Ok(None) + } + }; + let displaced = match displaced { + Ok(displaced) => displaced, + Err(held) => { + drop(table); + debug!("insert: flow {flow_key} is already held by a live flow"); + return Ok(Insertion::Occupied(held)); + } + }; + + val.update_status(FlowStatus::Active); + drop(table); + + #[cfg(not(any(feature = "shuttle", feature = "loom")))] + Self::start_timer(self.table.clone(), val.clone()); + + Self::displace(displaced.as_ref()); + + Ok(Insertion::Installed) + } + /// Lookup a flow in the table. /// /// # Panics @@ -412,6 +471,7 @@ mod tests { #[concurrency_mode(std)] mod std_tests { + use net::flows::FlowInfoFlags; use std::time::Instant; use tracing_test::traced_test; @@ -634,6 +694,91 @@ mod tests { assert_eq!(flow_table.active_len().unwrap(), 0); } + fn key_for(src_port: u16) -> FlowKey { + FlowKey::new( + Some(VpcDiscriminant::VNI(Vni::new_checked(1).unwrap())), + "1.2.3.4".parse::().unwrap(), + "4.5.6.7".parse::().unwrap(), + IpProtoKey::Tcp(TcpProtoKey { + src_port: TcpPort::new_checked(src_port).unwrap(), + dst_port: TcpPort::new_checked(2048).unwrap(), + }), + ) + } + + #[tokio::test] + async fn an_active_flow_holds_its_key_against_a_second_insertion() { + let flow_table = FlowTable::default(); + let key = key_for(1025); + let far_future = Instant::now() + Duration::from_hours(1); + + let first = Arc::new(FlowInfo::new(key, far_future)); + assert!(matches!( + flow_table.insert_if_absent(&first).unwrap(), + Insertion::Installed + )); + + let second = Arc::new(FlowInfo::new(key, far_future)); + let outcome = flow_table.insert_if_absent(&second).unwrap(); + let Insertion::Occupied(held) = outcome else { + panic!("a live flow was displaced by a second insertion: {outcome:?}"); + }; + assert!(Arc::ptr_eq(&held, &first), "the wrong flow was reported"); + + let found = flow_table.lookup(&key).expect("the key is still served"); + assert!(Arc::ptr_eq(&found, &first)); + assert_ne!(second.status(), FlowStatus::Active); + } + + #[tokio::test] + async fn a_flow_that_is_not_live_is_displaced() { + let flow_table = FlowTable::default(); + let key = key_for(1026); + let far_future = Instant::now() + Duration::from_hours(1); + + let first = Arc::new(FlowInfo::new(key, far_future)); + flow_table.insert_if_absent(&first).unwrap(); + first.invalidate(); + + let second = Arc::new(FlowInfo::new(key, far_future)); + assert!( + matches!( + flow_table.insert_if_absent(&second).unwrap(), + Insertion::Installed + ), + "a flow that was no longer live held its key" + ); + let found = flow_table.lookup(&key).expect("the key is served"); + assert!(Arc::ptr_eq(&found, &second)); + } + + #[tokio::test] + async fn displacing_a_flow_invalidates_its_partner() { + let flow_table = FlowTable::default(); + let (forward_key, reverse_key) = (key_for(1027), key_for(1028)); + let far_future = Instant::now() + Duration::from_hours(1); + + let (forward, reverse) = FlowInfo::related_pair( + far_future, + forward_key, + FlowInfoFlags::INITIATOR, + reverse_key, + FlowInfoFlags::default(), + ) + .expect("related_pair should succeed for distinct keys"); + flow_table.insert_from_arc(&forward).unwrap(); + flow_table.insert_from_arc(&reverse).unwrap(); + assert!(reverse.is_active()); + + let replacement = Arc::new(FlowInfo::new(forward_key, far_future)); + flow_table.insert_from_arc(&replacement).unwrap(); + + assert!( + !reverse.is_active(), + "the partner of a displaced flow was left live in the table" + ); + } + #[tokio::test] async fn test_flow_table_capacity_exceeded() { let flow_table = FlowTable::default(); diff --git a/k8s-intf/src/bolero/acl.rs b/k8s-intf/src/bolero/acl.rs new file mode 100644 index 0000000000..3cc3d528ca --- /dev/null +++ b/k8s-intf/src/bolero/acl.rs @@ -0,0 +1,229 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Open Network Fabric Authors + +use std::ops::Bound; + +use bolero::{Driver, ValueGenerator}; + +use crate::gateway_agent_crd::{ + GatewayAgentPeeringsAcl, GatewayAgentPeeringsAclDefault, GatewayAgentPeeringsAclRules, + GatewayAgentPeeringsAclRulesAction, GatewayAgentPeeringsAclRulesMatch, + GatewayAgentPeeringsAclRulesMatchDst, GatewayAgentPeeringsAclRulesMatchSrc, + GatewayAgentPeeringsAclRulesScope, GatewayAgentPeeringsPeering, +}; + +const MAX_RULES: u8 = 3; + +#[derive(Debug, Clone)] +pub struct SideFacts { + pub vpc: String, + pub native: Vec, + pub advertised: Vec, + pub restricts_ports: bool, + pub all_stateful: bool, +} + +impl SideFacts { + #[must_use] + pub fn of(vpc: &str, manifest: &GatewayAgentPeeringsPeering) -> Self { + let exposes = manifest.expose.as_deref().unwrap_or(&[]); + let mut native = Vec::new(); + let mut advertised = Vec::new(); + let mut restricts_ports = false; + let mut all_stateful = !exposes.is_empty(); + + for expose in exposes { + let ips: Vec = expose + .ips + .iter() + .flatten() + .filter_map(|ip| ip.cidr.clone()) + .collect(); + let translations: Vec = expose + .r#as + .iter() + .flatten() + .filter_map(|entry| entry.cidr.clone()) + .collect(); + + native.extend(ips.iter().cloned()); + if translations.is_empty() { + advertised.extend(ips); + } else { + advertised.extend(translations); + } + + let nat = expose.nat.as_ref(); + if nat.is_some_and(|nat| nat.port_forward.is_some()) { + restricts_ports = true; + } + if !nat.is_some_and(|nat| nat.masquerade.is_some() || nat.port_forward.is_some()) { + all_stateful = false; + } + } + + Self { + vpc: vpc.to_string(), + native, + advertised, + restricts_ports, + all_stateful, + } + } +} + +#[derive(Debug, Clone)] +pub struct AclGenerator { + left: SideFacts, + right: SideFacts, +} + +impl AclGenerator { + #[must_use] + pub fn new(left: SideFacts, right: SideFacts) -> Self { + Self { left, right } + } + + fn ports(d: &mut D) -> Option> { + let count = d.gen_u8(Bound::Included(&1), Bound::Included(&2))?; + let mut out = Vec::with_capacity(usize::from(count)); + for _ in 0..count { + let first = d.gen_u16(Bound::Included(&1), Bound::Included(&65535))?; + if d.produce::()? { + out.push(format!("{first}")); + } else { + let second = d.gen_u16(Bound::Included(&1), Bound::Included(&65535))?; + out.push(format!("{}-{}", first.min(second), first.max(second))); + } + } + Some(out) + } + + fn prefix(d: &mut D, choices: &[String]) -> Option { + if choices.is_empty() { + return None; + } + let index = d.gen_usize(Bound::Included(&0), Bound::Excluded(&choices.len()))?; + Some(choices[index].clone()) + } + + fn rule(&self, d: &mut D, index: u8) -> Option { + let (from, to) = if d.produce::()? { + (&self.left, &self.right) + } else { + (&self.right, &self.left) + }; + + let (from_field, to_field) = match d.gen_u8(Bound::Included(&0), Bound::Included(&2))? { + 0 => (Some(from.vpc.clone()), None), + 1 => (None, Some(to.vpc.clone())), + _ => (Some(from.vpc.clone()), Some(to.vpc.clone())), + }; + + let (proto, may_use_ports) = match d.gen_u8(Bound::Included(&0), Bound::Included(&3))? { + 0 => (None, false), + 1 => (Some("tcp".to_string()), true), + 2 => (Some("udp".to_string()), true), + _ => ( + Some( + d.gen_u8(Bound::Included(&1), Bound::Included(&254))? + .to_string(), + ), + false, + ), + }; + + let src_ports = may_use_ports && !from.restricts_ports && d.produce::()?; + let dst_ports = may_use_ports && !to.restricts_ports && d.produce::()?; + + let src_prefix = if d.produce::()? { + Self::prefix(d, &from.native) + } else { + None + }; + let dst_prefix = if d.produce::()? { + Self::prefix(d, &to.advertised) + } else { + None + }; + + let src = if src_prefix.is_some() || src_ports { + Some(vec![GatewayAgentPeeringsAclRulesMatchSrc { + cidr: src_prefix, + ports: if src_ports { + Self::ports(d)?.into() + } else { + None + }, + vpc_subnet: None, + }]) + } else { + None + }; + let dst = if dst_prefix.is_some() || dst_ports { + Some(vec![GatewayAgentPeeringsAclRulesMatchDst { + cidr: dst_prefix, + ports: if dst_ports { + Self::ports(d)?.into() + } else { + None + }, + vpc_subnet: None, + }]) + } else { + None + }; + + let r#match = if src.is_some() || dst.is_some() || proto.is_some() { + Some(GatewayAgentPeeringsAclRulesMatch { dst, proto, src }) + } else { + None + }; + + let flow_allowed = self.left.all_stateful || self.right.all_stateful; + let scope = if flow_allowed && d.produce::()? { + if d.produce::()? { + Some(GatewayAgentPeeringsAclRulesScope::Flow) + } else { + None + } + } else { + Some(GatewayAgentPeeringsAclRulesScope::Packet) + }; + + Some(GatewayAgentPeeringsAclRules { + action: if d.produce::()? { + GatewayAgentPeeringsAclRulesAction::Allow + } else { + GatewayAgentPeeringsAclRulesAction::Deny + }, + from: from_field, + log: Some(d.produce::()?), + r#match, + name: Some(format!("rule{index}")), + scope, + to: to_field, + }) + } +} + +impl ValueGenerator for AclGenerator { + type Output = GatewayAgentPeeringsAcl; + + fn generate(&self, d: &mut D) -> Option { + let count = d.gen_u8(Bound::Included(&1), Bound::Included(&MAX_RULES))?; + let mut rules = Vec::with_capacity(usize::from(count)); + for index in 0..count { + rules.push(self.rule(d, index)?); + } + + Some(GatewayAgentPeeringsAcl { + default: match d.gen_u8(Bound::Included(&0), Bound::Included(&2))? { + 0 => GatewayAgentPeeringsAclDefault::Deny, + 1 => GatewayAgentPeeringsAclDefault::DenyUnlessExposed, + _ => GatewayAgentPeeringsAclDefault::KopiumEmpty, + }, + rules: Some(rules).filter(|r| !r.is_empty()), + }) + } +} diff --git a/k8s-intf/src/bolero/crd.rs b/k8s-intf/src/bolero/crd.rs index 92fc069b5a..532f646502 100644 --- a/k8s-intf/src/bolero/crd.rs +++ b/k8s-intf/src/bolero/crd.rs @@ -6,7 +6,8 @@ use std::ops::Bound; use bolero::{Driver, TypeGenerator, ValueGenerator, produce}; use kube::core::ObjectMeta; -use crate::bolero::LegalValue; +use crate::bolero::spec::{GatewayAgentSpecs, SpecBuilder}; +use crate::bolero::{AddressFamily, LegalValue, NatFlavour}; use crate::gateway_agent_crd::{GatewayAgent, GatewayAgentSpec}; const HOSTNAME_BASE: &str = "host-"; @@ -23,21 +24,86 @@ fn simple_hostname(d: &mut D) -> Option { ) } +fn join_own_groups(d: &mut D, name: &str, spec: &mut GatewayAgentSpec) -> Option<()> { + for group in spec.groups.iter_mut().flatten().map(|(_, group)| group) { + if !d.produce::()? { + continue; + } + if let Some(member) = group.members.iter_mut().flatten().next() { + member.name = name.to_string(); + } + } + Some(()) +} + +#[derive(Debug, Clone, Default)] +pub struct GatewayAgents(GatewayAgentSpecs); + +impl ValueGenerator for GatewayAgents { + type Output = GatewayAgent; + + fn generate(&self, d: &mut D) -> Option { + let name = simple_hostname(d)?; + let generation = d.gen_i64(Bound::Excluded(&0), Bound::Unbounded)?; + let mut spec = self.0.generate(d)?; + join_own_groups(d, &name, &mut spec)?; + Some(GatewayAgent { + metadata: ObjectMeta { + name: Some(name), + generation: Some(generation), + namespace: Some("default".to_string()), + ..Default::default() + }, + spec, + status: None, // Add when we build a generator and converter for status + }) + } +} + +#[derive(Debug, Clone, Default)] +pub struct GatewayAgentBuilder(SpecBuilder); + +impl GatewayAgentBuilder { + #[must_use] + pub fn new() -> Self { + Self::default() + } + + #[must_use] + pub fn flavours(mut self, flavours: Vec) -> Self { + self.0 = self.0.flavours(flavours); + self + } + + #[must_use] + pub fn families(mut self, families: Vec) -> Self { + self.0 = self.0.families(families); + self + } + + #[must_use] + pub fn sizes(mut self, vpcs: u8, peerings: u8, exposes: u8, subnets: u8) -> Self { + self.0 = self + .0 + .max_vpcs(vpcs) + .max_peerings(peerings) + .max_exposes(exposes) + .max_subnets(subnets); + self + } + + #[must_use] + pub fn build(self) -> GatewayAgents { + GatewayAgents(self.0.build()) + } +} + /// Generate a random legal `GatewayAgent` value /// /// Is not exhaustive due to hostname generation /// Coverage of values is subject to limitations of the `GatewayAgentSpec` `TypeGenerator` as well impl TypeGenerator for LegalValue { fn generate(d: &mut D) -> Option { - Some(LegalValue(GatewayAgent { - metadata: ObjectMeta { - name: Some(simple_hostname(d)?), - generation: Some(d.gen_i64(Bound::Excluded(&0), Bound::Unbounded)?), - namespace: Some("default".to_string()), - ..Default::default() - }, - spec: d.produce::>()?.take(), - status: None, // Add when we build a generator and converter for status - })) + Some(LegalValue(GatewayAgents::default().generate(d)?)) } } diff --git a/k8s-intf/src/bolero/expose.rs b/k8s-intf/src/bolero/expose.rs index 8750ed9c29..f5e7436e2a 100644 --- a/k8s-intf/src/bolero/expose.rs +++ b/k8s-intf/src/bolero/expose.rs @@ -3,10 +3,10 @@ use std::ops::Bound; -use bolero::{Driver, TypeGenerator, ValueGenerator}; +use bolero::{Driver, ValueGenerator}; -use crate::bolero::support::generate_prefixes; -use crate::bolero::{LegalValue, SubnetMap}; +use crate::bolero::support::blocks; +use crate::bolero::{AddressFamily, NatFlavour, SubnetMap}; use crate::gateway_agent_crd::{ GatewayAgentPeeringsPeeringExpose, GatewayAgentPeeringsPeeringExposeAs, GatewayAgentPeeringsPeeringExposeIps, GatewayAgentPeeringsPeeringExposeNat, @@ -17,175 +17,281 @@ use crate::gateway_agent_crd::{ GatewayAgentPeeringsPeeringExposeNatStatic, }; -/// Generate a legal value for `GatewayAgentPeeringsPeeringExpose` /// -/// This is not exhaustive over all legal values due to the complexity of doing this. For example, -/// the CIDR generators are not exhaustive; and we use a single port range for all CIDRs rather than -/// trying different combinations. -pub struct LegalValueExposeGenerator<'a> { +const MAX_PREFIXES: u8 = 3; + +const MAX_PORTS: u16 = 1024; + +#[derive(Debug, Clone)] +pub struct ExposeGenerator<'a> { + flavour: NatFlavour, + family: AddressFamily, + which: Which, subnets: &'a SubnetMap, } -impl<'a> LegalValueExposeGenerator<'a> { +#[derive(Debug, Clone, Copy)] +pub struct Which { + pub slot: u8, + pub index: u8, + pub count: u8, +} + +impl Which { #[must_use] - pub fn new(subnets: &'a SubnetMap) -> Self { - Self { subnets } + pub fn only(slot: u8) -> Self { + Self { + slot, + index: 0, + count: 1, + } + } + + #[must_use] + pub fn nth(slot: u8, index: u8, count: u8) -> Self { + Self { + slot, + index, + count: count.max(index.saturating_add(1)), + } } } -impl ValueGenerator for LegalValueExposeGenerator<'_> { - type Output = GatewayAgentPeeringsPeeringExpose; +impl<'a> ExposeGenerator<'a> { + #[must_use] + pub fn new( + flavour: NatFlavour, + family: AddressFamily, + which: Which, + subnets: &'a SubnetMap, + ) -> Self { + Self { + flavour, + family, + which, + subnets, + } + } - fn generate(&self, d: &mut D) -> Option { - let num_ips = d.gen_u16(Bound::Included(&1), Bound::Included(&16))?; - let num_nots = d.gen_u16(Bound::Included(&0), Bound::Included(&16))?; - let num_subnets = std::cmp::max( - self.subnets.len(), - d.gen_usize(Bound::Included(&0), Bound::Included(&16))?, - ); - - let num_as = d.gen_u16(Bound::Included(&0), Bound::Included(&16))?; - let num_as_not = d.gen_u16(Bound::Included(&0), Bound::Included(&16))?; - - let num_v4_ips = d.gen_u16(Bound::Included(&0), Bound::Included(&num_ips))?; - let num_v6_ips = num_ips - num_v4_ips; - let num_v4_nots = d.gen_u16(Bound::Included(&0), Bound::Included(&num_nots))?; - let num_v6_nots = num_nots - num_v4_nots; - - let num_v4_as = d.gen_u16(Bound::Included(&0), Bound::Included(&num_as))?; - let num_v6_as = num_as - num_v4_as; - let num_v4_not_as = d.gen_u16(Bound::Included(&0), Bound::Included(&num_as_not))?; - let num_v6_not_as = num_as_not - num_v4_not_as; - - let ips = generate_prefixes(d, num_v4_ips, num_v6_ips)? - .into_iter() - .map(|p| GatewayAgentPeeringsPeeringExposeIps { - cidr: Some(p), - not: None, - vpc_subnet: None, - }) - .collect::>(); - let nots = generate_prefixes(d, num_v4_nots, num_v6_nots)? - .into_iter() - .map(|p| GatewayAgentPeeringsPeeringExposeIps { - cidr: None, - not: Some(p), - vpc_subnet: None, + fn length(&self, d: &mut D, at: blocks::At) -> Option { + d.gen_u8( + Bound::Included(&blocks::min_len_at(self.family, at)), + Bound::Included(&blocks::max_len(self.family)), + ) + } + + fn matching_subnets(&self) -> Vec<&'a String> { + self.subnets + .iter() + .filter(|(_, prefix)| prefix.is_ipv4() == self.family.is_v4()) + .map(|(name, _)| name) + .enumerate() + .filter(|(index, _)| { + index % usize::from(self.which.count) == usize::from(self.which.index) }) - .collect::>(); - let r#as = generate_prefixes(d, num_v4_as, num_v6_as)? - .into_iter() - .map(|p| GatewayAgentPeeringsPeeringExposeAs { - cidr: Some(p), - not: None, - }); - let not_as = generate_prefixes(d, num_v4_not_as, num_v6_not_as)? - .into_iter() - .map(|p| GatewayAgentPeeringsPeeringExposeAs { - cidr: None, - not: Some(p), - }); + .map(|(_, name)| name) + .collect() + } - let mut subnets = Vec::new(); - let mut subnet_iter = self.subnets.iter(); - for _ in 0..num_subnets { - let Some((name, _)) = subnet_iter.next() else { - break; - }; - subnets.push(GatewayAgentPeeringsPeeringExposeIps { - cidr: None, - not: None, - vpc_subnet: Some(name.clone()), - }); + fn exclusion( + &self, + d: &mut D, + parent: &str, + at: blocks::At, + private: bool, + ) -> Option { + let (_, len) = parent.split_once('/')?; + let len: u8 = len.parse().ok()?; + let max = blocks::max_len(self.family); + if len >= max { + return None; } + let longer = d.gen_u8(Bound::Excluded(&len), Bound::Included(&max))?; + if private { + blocks::private(d, self.family, at, longer) + } else { + blocks::public(d, self.family, at, longer) + } + } - let mut final_ips = Vec::with_capacity(ips.len() + nots.len() + subnets.len()); - final_ips.extend(ips); - final_ips.extend(nots); - final_ips.extend(subnets); - - let mut final_as = Vec::with_capacity(r#as.len() + not_as.len()); - final_as.extend(r#as); - final_as.extend(not_as); - let has_as = !final_as.is_empty(); - - Some(GatewayAgentPeeringsPeeringExpose { - r#as: Some(final_as).filter(|f| !f.is_empty()), - ips: Some(final_ips).filter(|f| !f.is_empty()), - default: None, - nat: if has_as { - Some( - d.produce::>()? - .take(), - ) - } else { - None - }, - }) + fn port_pair(d: &mut D) -> Option<(String, String)> { + let size = d.gen_u16(Bound::Included(&1), Bound::Included(&MAX_PORTS))?; + let first_start = d.gen_u16(Bound::Included(&1), Bound::Included(&(65535 - size + 1)))?; + let second_start = d.gen_u16(Bound::Included(&1), Bound::Included(&(65535 - size + 1)))?; + Some(( + format!("{first_start}-{}", first_start + (size - 1)), + format!("{second_start}-{}", second_start + (size - 1)), + )) } -} -// This is not exhaustive as it does not generate all possible time -// strings, just 0 to 2*3600 seconds. -// -impl TypeGenerator for LegalValue { - fn generate(d: &mut D) -> Option { - let nat_mode = d.produce::()? % 3; - let idle_timeout_secs = d.gen_u64(Bound::Included(&0), Bound::Included(&(2 * 3600)))?; - let idle_timeout = std::time::Duration::from_secs(idle_timeout_secs); - match nat_mode { - 0 => Some(LegalValue(GatewayAgentPeeringsPeeringExposeNat { + fn translation(&self, d: &mut D) -> Option { + let idle_secs = d.gen_u64(Bound::Included(&0), Bound::Included(&(2 * 3600)))?; + let idle = std::time::Duration::from_secs(idle_secs); + Some(match self.flavour { + NatFlavour::None => return None, + NatFlavour::Masquerade => GatewayAgentPeeringsPeeringExposeNat { masquerade: Some(GatewayAgentPeeringsPeeringExposeNatMasquerade { - idle_timeout: Some(idle_timeout.into()), + idle_timeout: Some(idle.into()), }), port_forward: None, r#static: None, - })), - 1 => Some(LegalValue(GatewayAgentPeeringsPeeringExposeNat { + }, + NatFlavour::Static => GatewayAgentPeeringsPeeringExposeNat { masquerade: None, port_forward: None, r#static: Some(GatewayAgentPeeringsPeeringExposeNatStatic {}), - })), - 2 => { - // Generate a valid port range - let bound1 = d.gen_u16(Bound::Included(&1), Bound::Included(&65535))?; - let bound2 = d.gen_u16(Bound::Included(&1), Bound::Included(&65535))?; - let start = bound1.min(bound2); - let end = bound1.max(bound2); - let port_range = format!("{start}-{end}"); - - // Generate another valid port range of the same size - let port_range_size = (end - start) as usize + 1; - let max_new_start = u16::try_from(65536 - port_range_size).unwrap(); - let new_bound = d.gen_u16(Bound::Included(&0), Bound::Included(&max_new_start))?; - let new_port_range = format!( - "{new_bound}-{}", - new_bound + u16::try_from(port_range_size - 1).unwrap() - ); - - Some(LegalValue(GatewayAgentPeeringsPeeringExposeNat { + }, + NatFlavour::PortForward => { + let (port, r#as) = Self::port_pair(d)?; + GatewayAgentPeeringsPeeringExposeNat { masquerade: None, port_forward: Some(GatewayAgentPeeringsPeeringExposeNatPortForward { - idle_timeout: Some(idle_timeout.into()), + idle_timeout: Some(idle.into()), ports: Some(vec![GatewayAgentPeeringsPeeringExposeNatPortForwardPorts { - r#as: Some(new_port_range), - port: Some(port_range), - proto: match d.produce::()? % 3 { + r#as: Some(r#as), + port: Some(port), + proto: match d.gen_u8(Bound::Included(&0), Bound::Included(&2))? { 0 => Some( GatewayAgentPeeringsPeeringExposeNatPortForwardPortsProto::Tcp, ), 1 => Some( GatewayAgentPeeringsPeeringExposeNatPortForwardPortsProto::Udp, ), - 2 => None, - _ => unreachable!(), + _ => None, }, }]), }), r#static: None, - })) + } + } + }) + } +} + +impl ValueGenerator for ExposeGenerator<'_> { + type Output = GatewayAgentPeeringsPeeringExpose; + + fn generate(&self, d: &mut D) -> Option { + let paired = matches!(self.flavour, NatFlavour::Static | NatFlavour::PortForward); + + let mut ips = Vec::new(); + let mut translations = Vec::new(); + + if paired { + let at = blocks::At::whole(self.which.slot); + let len = self.length(d, at)?; + let private = blocks::private(d, self.family, at, len)?; + let public = blocks::public(d, self.family, at, len)?; + ips.push(GatewayAgentPeeringsPeeringExposeIps { + cidr: Some(private), + not: None, + vpc_subnet: None, + }); + translations.push(GatewayAgentPeeringsPeeringExposeAs { + cidr: Some(public), + not: None, + }); + } else { + let count = d.gen_u8(Bound::Included(&1), Bound::Included(&MAX_PREFIXES))?; + for sub in 0..count { + let at = blocks::At::nth(self.which.slot, sub, count); + let len = self.length(d, at)?; + ips.push(GatewayAgentPeeringsPeeringExposeIps { + cidr: Some(blocks::private(d, self.family, at, len)?), + not: None, + vpc_subnet: None, + }); + } + if self.flavour.needs_translation() { + let count = d.gen_u8(Bound::Included(&1), Bound::Included(&MAX_PREFIXES))?; + for sub in 0..count { + let at = blocks::At::nth(self.which.slot, sub, count); + let len = self.length(d, at)?; + translations.push(GatewayAgentPeeringsPeeringExposeAs { + cidr: Some(blocks::public(d, self.family, at, len)?), + not: None, + }); + } + } + + let named = self.matching_subnets(); + if !named.is_empty() { + let take = d.gen_usize(Bound::Included(&0), Bound::Included(&named.len()))?; + for name in named.into_iter().take(take) { + ips.push(GatewayAgentPeeringsPeeringExposeIps { + cidr: None, + not: None, + vpc_subnet: Some(name.clone()), + }); + } } - _ => unreachable!(), } + + if self.flavour.allows_exclusions() && d.produce::()? { + let parents: Vec = ips.iter().filter_map(|e| e.cidr.clone()).collect(); + let first = + blocks::At::nth(self.which.slot, 0, u8::try_from(parents.len()).unwrap_or(1)); + if let Some(parent) = parents.first() + && let Some(exclusion) = self.exclusion(d, parent, first, true) + { + ips.push(GatewayAgentPeeringsPeeringExposeIps { + cidr: None, + not: Some(exclusion), + vpc_subnet: None, + }); + } + let parents: Vec = translations.iter().filter_map(|e| e.cidr.clone()).collect(); + let first = + blocks::At::nth(self.which.slot, 0, u8::try_from(parents.len()).unwrap_or(1)); + if let Some(parent) = parents.first() + && let Some(exclusion) = self.exclusion(d, parent, first, false) + { + translations.push(GatewayAgentPeeringsPeeringExposeAs { + cidr: None, + not: Some(exclusion), + }); + } + } + + Some(GatewayAgentPeeringsPeeringExpose { + r#as: Some(translations).filter(|t| !t.is_empty()), + ips: Some(ips).filter(|i| !i.is_empty()), + default: None, + nat: if self.flavour.needs_translation() { + Some(self.translation(d)?) + } else { + None + }, + }) + } +} + +#[derive(Debug, Clone)] +pub struct AnyExposeGenerator<'a> { + which: Which, + subnets: &'a SubnetMap, +} + +impl<'a> AnyExposeGenerator<'a> { + #[must_use] + pub fn new(slot: u8, subnets: &'a SubnetMap) -> Self { + Self { + which: Which::only(slot), + subnets, + } + } +} + +impl ValueGenerator for AnyExposeGenerator<'_> { + type Output = GatewayAgentPeeringsPeeringExpose; + + fn generate(&self, d: &mut D) -> Option { + let flavours = NatFlavour::all(); + let families = AddressFamily::all(); + let flavour = + flavours[d.gen_usize(Bound::Included(&0), Bound::Excluded(&flavours.len()))?]; + let family = + families[d.gen_usize(Bound::Included(&0), Bound::Excluded(&families.len()))?]; + ExposeGenerator::new(flavour, family, self.which, self.subnets).generate(d) } } diff --git a/k8s-intf/src/bolero/mod.rs b/k8s-intf/src/bolero/mod.rs index 078ef00a18..e8234c1a72 100644 --- a/k8s-intf/src/bolero/mod.rs +++ b/k8s-intf/src/bolero/mod.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // Copyright Open Network Fabric Authors +pub mod acl; pub mod bgp; pub mod crd; pub mod expose; @@ -8,7 +9,10 @@ pub mod gateway; pub mod gwgroups; pub mod interface; pub mod logs; +pub mod mutate; pub mod peering; +pub mod permute; +pub mod reduce; pub mod spec; pub mod support; pub mod vpc; @@ -17,6 +21,59 @@ use std::collections::BTreeMap; use lpm::prefix::Prefix; +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum NatFlavour { + None, + Masquerade, + Static, + PortForward, +} + +impl NatFlavour { + #[must_use] + pub fn all() -> Vec { + vec![ + Self::None, + Self::Masquerade, + Self::Static, + Self::PortForward, + ] + } + + #[must_use] + pub fn allows_exclusions(self) -> bool { + matches!(self, Self::None | Self::Masquerade) + } + + #[must_use] + pub fn needs_translation(self) -> bool { + !matches!(self, Self::None) + } + + #[must_use] + pub fn is_stateful(self) -> bool { + matches!(self, Self::Masquerade | Self::PortForward) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum AddressFamily { + V4, + V6, +} + +impl AddressFamily { + #[must_use] + pub fn all() -> Vec { + vec![Self::V4, Self::V6] + } + + #[must_use] + pub fn is_v4(self) -> bool { + matches!(self, Self::V4) + } +} + /// A type on which implement `bolero::TypeGenerator` for legal values of `T` /// /// Generally, `bolero` type generators should generate all possible values of `T` so that it is possible to test validation logic, etc. @@ -70,9 +127,9 @@ where // This is distinct from the SubnetMap in config/converters/k8s // since this type is only for the test library. It should be // compatible with the SubnetMap in config/converters/k8s -type SubnetMap = BTreeMap; +pub(crate) type SubnetMap = BTreeMap; // This is distinct from the VpcSubnetMap in config/converters/k8s // since this type is only for the test library. It should be // compatible with the SubnetMap in config/converters/k8s -type VpcSubnetMap = BTreeMap; +pub(crate) type VpcSubnetMap = BTreeMap; diff --git a/k8s-intf/src/bolero/mutate.rs b/k8s-intf/src/bolero/mutate.rs new file mode 100644 index 0000000000..9e70ebf8ed --- /dev/null +++ b/k8s-intf/src/bolero/mutate.rs @@ -0,0 +1,527 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Open Network Fabric Authors + +use std::ops::Bound; + +use bolero::{Driver, ValueGenerator}; + +use crate::bolero::crd::GatewayAgents; +use crate::gateway_agent_crd::{ + GatewayAgent, GatewayAgentPeeringsAclRulesScope, GatewayAgentPeeringsPeering, + GatewayAgentPeeringsPeeringExpose, GatewayAgentPeeringsPeeringExposeAs, + GatewayAgentPeeringsPeeringExposeIps, GatewayAgentPeeringsPeeringExposeNatMasquerade, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Mutation { + None, + MismatchPortForwardPrefixes, + MismatchStaticNatPrefixes, + ExcludeFromPortForwarding, + MixAddressFamilies, + UseReservedPrefix, + EmptyPrivatePrefixes, + DropTranslationRange, + MakeBothSidesStateful, + NameAMissingGroup, + NameAStrangerInARule, + DemandFlowScope, + UsePortZero, + DuplicateAStaticExpose, + OverlapWithAnotherPeer, +} + +impl Mutation { + pub const COUNT: usize = 15; + + #[must_use] + pub fn index(self) -> usize { + Self::all() + .iter() + .position(|other| *other == self) + .unwrap_or_else(|| unreachable!()) + } + + #[must_use] + pub fn all() -> Vec { + vec![ + Self::None, + Self::MismatchPortForwardPrefixes, + Self::MismatchStaticNatPrefixes, + Self::ExcludeFromPortForwarding, + Self::MixAddressFamilies, + Self::UseReservedPrefix, + Self::EmptyPrivatePrefixes, + Self::DropTranslationRange, + Self::MakeBothSidesStateful, + Self::NameAMissingGroup, + Self::NameAStrangerInARule, + Self::DemandFlowScope, + Self::UsePortZero, + Self::DuplicateAStaticExpose, + Self::OverlapWithAnotherPeer, + ] + } +} + +fn lengthen(cidr: &str, by: u8) -> Option { + let (address, len) = cidr.split_once('/')?; + let len: u8 = len.parse().ok()?; + let max = if address.contains(':') { 128 } else { 32 }; + let longer = len.saturating_add(by).min(max); + if longer == len { + return None; + } + Some(format!("{address}/{longer}")) +} + +fn sibling(cidr: &str) -> Option { + let (address, len) = cidr.split_once('/')?; + let len: u8 = len.parse().ok()?; + if address.contains(':') { + let bits = address.parse::().ok()?.to_bits(); + let flipped = bits ^ (1u128 << (128u8.checked_sub(len)?)); + Some(format!("{}/{len}", std::net::Ipv6Addr::from(flipped))) + } else { + let bits = address.parse::().ok()?.to_bits(); + let flipped = bits ^ (1u32 << (32u8.checked_sub(len)?)); + Some(format!("{}/{len}", std::net::Ipv4Addr::from(flipped))) + } +} + +fn exposes_mut(agent: &mut GatewayAgent) -> Vec<&mut GatewayAgentPeeringsPeeringExpose> { + agent + .spec + .peerings + .iter_mut() + .flatten() + .flat_map(|(_, peerings)| peerings.peering.iter_mut().flatten()) + .flat_map(|(_, manifest)| manifest.expose.iter_mut().flatten()) + .collect() +} + +fn is_port_forwarding(expose: &GatewayAgentPeeringsPeeringExpose) -> bool { + expose + .nat + .as_ref() + .is_some_and(|nat| nat.port_forward.is_some()) +} + +fn is_static(expose: &GatewayAgentPeeringsPeeringExpose) -> bool { + expose + .nat + .as_ref() + .is_some_and(|nat| nat.r#static.is_some()) +} + +fn stateful_throughout(manifest: &GatewayAgentPeeringsPeering) -> bool { + let exposes = manifest.expose.as_deref().unwrap_or(&[]); + !exposes.is_empty() + && exposes.iter().all(|expose| { + expose + .nat + .as_ref() + .is_some_and(|nat| nat.masquerade.is_some() || nat.port_forward.is_some()) + }) +} + +fn advertises_its_ips(expose: &GatewayAgentPeeringsPeeringExpose) -> bool { + expose.nat.is_none() + && expose.default != Some(true) + && expose.ips.iter().flatten().all(|ip| ip.not.is_none()) +} + +fn overlap_with_another_peer(agent: &mut GatewayAgent) -> bool { + let mut donor: Option<(String, String, String)> = None; + for (key, peering) in agent.spec.peerings.iter().flatten() { + let Some(manifests) = peering.peering.as_ref() else { + continue; + }; + for shared in manifests.keys() { + let elsewhere = agent + .spec + .peerings + .iter() + .flatten() + .any(|(other, peering)| { + other != key + && peering + .peering + .as_ref() + .is_some_and(|m| m.contains_key(shared)) + }); + if !elsewhere { + continue; + } + let prefix = manifests + .iter() + .filter(|(name, _)| *name != shared) + .flat_map(|(_, manifest)| manifest.expose.iter().flatten()) + .filter(|expose| advertises_its_ips(expose)) + .flat_map(|expose| expose.ips.iter().flatten()) + .find_map(|ip| ip.cidr.clone()); + if let Some(prefix) = prefix { + donor = Some((key.clone(), shared.clone(), prefix)); + break; + } + } + if donor.is_some() { + break; + } + } + + let mut done = false; + if let Some((donor_key, shared, prefix)) = donor { + for (key, peering) in agent.spec.peerings.iter_mut().flatten() { + if *key == donor_key { + continue; + } + let Some(manifests) = peering.peering.as_mut() else { + continue; + }; + if !manifests.contains_key(&shared) { + continue; + } + let names: Vec = manifests + .keys() + .filter(|name| **name != shared) + .cloned() + .collect(); + for name in names { + let Some(manifest) = manifests.get_mut(&name) else { + continue; + }; + if let Some(ip) = manifest + .expose + .iter_mut() + .flatten() + .filter(|expose| advertises_its_ips(expose)) + .flat_map(|expose| expose.ips.iter_mut().flatten()) + .find(|ip| ip.cidr.is_some()) + { + ip.cidr = Some(prefix.clone()); + done = true; + break; + } + } + if done { + break; + } + } + } + done +} + +fn mutate_peering_metadata(agent: &mut GatewayAgent, mutation: Mutation) -> bool { + match mutation { + Mutation::MakeBothSidesStateful => { + let mut done = false; + for (_, peerings) in agent.spec.peerings.iter_mut().flatten() { + let manifests = peerings.peering.iter_mut().flatten(); + let mut touched = 0; + for (_, manifest) in manifests { + for expose in manifest.expose.iter_mut().flatten() { + let nat = expose.nat.get_or_insert( + crate::gateway_agent_crd::GatewayAgentPeeringsPeeringExposeNat { + masquerade: None, + port_forward: None, + r#static: None, + }, + ); + nat.port_forward = None; + nat.r#static = None; + nat.masquerade = Some(GatewayAgentPeeringsPeeringExposeNatMasquerade { + idle_timeout: None, + }); + if expose.r#as.is_none() { + expose.r#as = Some(vec![GatewayAgentPeeringsPeeringExposeAs { + cidr: Some("172.31.0.0/16".to_string()), + not: None, + }]); + } + } + touched += 1; + } + if touched == 2 { + done = true; + break; + } + } + done + } + + Mutation::NameAMissingGroup => { + if let Some((_, peerings)) = agent.spec.peerings.iter_mut().flatten().next() { + peerings.gateway_group = Some("no-such-group".to_string()); + true + } else { + false + } + } + + Mutation::NameAStrangerInARule => { + let mut done = false; + for (_, peerings) in agent.spec.peerings.iter_mut().flatten() { + let Some(acl) = peerings.acl.as_mut() else { + continue; + }; + if let Some(rule) = acl.rules.iter_mut().flatten().next() { + rule.from = Some("not-in-this-peering".to_string()); + done = true; + break; + } + } + done + } + + Mutation::DemandFlowScope => { + let mut done = false; + for (_, peerings) in agent.spec.peerings.iter_mut().flatten() { + let allowed = peerings + .peering + .iter() + .flatten() + .any(|(_, manifest)| stateful_throughout(manifest)); + if allowed { + continue; + } + let Some(acl) = peerings.acl.as_mut() else { + continue; + }; + for rule in acl.rules.iter_mut().flatten() { + rule.scope = Some(GatewayAgentPeeringsAclRulesScope::Flow); + done = true; + } + if done { + break; + } + } + done + } + _ => unreachable!("mutate_peering_metadata called for {mutation:?}"), + } +} + +fn mutate_expose_shape(agent: &mut GatewayAgent, mutation: Mutation) -> bool { + match mutation { + Mutation::MismatchPortForwardPrefixes | Mutation::MismatchStaticNatPrefixes => { + let wanted: fn(&GatewayAgentPeeringsPeeringExpose) -> bool = + if mutation == Mutation::MismatchPortForwardPrefixes { + is_port_forwarding + } else { + is_static + }; + let mut done = false; + for expose in exposes_mut(agent) { + if !wanted(expose) { + continue; + } + if let Some(entry) = expose.r#as.iter_mut().flatten().next() + && let Some(cidr) = entry.cidr.as_ref() + && let Some(longer) = lengthen(cidr, 2) + { + entry.cidr = Some(longer); + done = true; + break; + } + } + done + } + + Mutation::ExcludeFromPortForwarding => { + let mut done = false; + for expose in exposes_mut(agent) { + if !is_port_forwarding(expose) { + continue; + } + let Some(inside) = expose + .ips + .iter() + .flatten() + .find_map(|ip| ip.cidr.as_ref()) + .and_then(|cidr| lengthen(cidr, 1)) + else { + continue; + }; + expose.ips.get_or_insert_with(Vec::new).push( + GatewayAgentPeeringsPeeringExposeIps { + cidr: None, + not: Some(inside), + vpc_subnet: None, + }, + ); + done = true; + break; + } + done + } + + Mutation::MixAddressFamilies => { + let mut done = false; + for expose in exposes_mut(agent) { + let Some(entry) = expose.ips.iter_mut().flatten().next() else { + continue; + }; + let Some(cidr) = entry.cidr.as_ref() else { + continue; + }; + let other = if cidr.contains(':') { + "10.99.0.0/16" + } else { + "2001:db8:9999::/48" + }; + expose.ips.get_or_insert_with(Vec::new).push( + GatewayAgentPeeringsPeeringExposeIps { + cidr: Some(other.to_string()), + not: None, + vpc_subnet: None, + }, + ); + done = true; + break; + } + done + } + _ => unreachable!("mutate_expose_shape called for {mutation:?}"), + } +} + +fn duplicate_a_static_expose(agent: &mut GatewayAgent) -> bool { + for (_, peering) in agent.spec.peerings.iter_mut().flatten() { + for manifest in peering.peering.iter_mut().flatten().map(|(_, m)| m) { + let Some(exposes) = manifest.expose.as_mut() else { + continue; + }; + let statics: Vec = exposes + .iter() + .enumerate() + .filter(|(_, expose)| { + expose + .nat + .as_ref() + .is_some_and(|nat| nat.r#static.is_some()) + }) + .map(|(index, _)| index) + .collect(); + + let (Some(donor), Some(recipient)) = (statics.first(), statics.get(1)) else { + continue; + }; + let mut copy = exposes[*donor].clone(); + let moved = copy + .r#as + .as_mut() + .and_then(|ranges| ranges.first_mut()) + .and_then(|range| { + let next_door = sibling(range.cidr.as_ref()?)?; + range.cidr = Some(next_door); + Some(()) + }); + if moved.is_some() { + exposes[*recipient] = copy; + return true; + } + } + } + false +} + +pub fn apply(d: &mut D, agent: &mut GatewayAgent, mutation: Mutation) -> Option { + let bit = match mutation { + Mutation::None => false, + + Mutation::MismatchPortForwardPrefixes + | Mutation::MismatchStaticNatPrefixes + | Mutation::ExcludeFromPortForwarding + | Mutation::MixAddressFamilies => mutate_expose_shape(agent, mutation), + Mutation::UseReservedPrefix => { + let reserved = ["127.0.0.0/8", "224.0.0.0/4", "0.0.0.0/8", "ff00::/8"]; + let choice = + reserved[d.gen_usize(Bound::Included(&0), Bound::Excluded(&reserved.len()))?]; + let mut done = false; + for expose in exposes_mut(agent) { + if let Some(entry) = expose.ips.iter_mut().flatten().next() + && entry.cidr.is_some() + { + entry.cidr = Some(choice.to_string()); + done = true; + break; + } + } + done + } + + Mutation::EmptyPrivatePrefixes => { + let mut done = false; + for expose in exposes_mut(agent) { + if expose.ips.is_some() { + expose.ips = None; + done = true; + break; + } + } + done + } + + Mutation::DropTranslationRange => { + let mut done = false; + for expose in exposes_mut(agent) { + if expose.nat.is_some() && expose.r#as.is_some() { + expose.r#as = None; + done = true; + break; + } + } + done + } + + Mutation::MakeBothSidesStateful + | Mutation::NameAMissingGroup + | Mutation::NameAStrangerInARule + | Mutation::DemandFlowScope => mutate_peering_metadata(agent, mutation), + Mutation::UsePortZero => { + let mut done = false; + for expose in exposes_mut(agent) { + let Some(nat) = expose.nat.as_mut() else { + continue; + }; + let Some(pf) = nat.port_forward.as_mut() else { + continue; + }; + if let Some(ports) = pf.ports.iter_mut().flatten().next() { + ports.port = Some("0-100".to_string()); + ports.r#as = Some("0-100".to_string()); + done = true; + break; + } + } + done + } + + Mutation::DuplicateAStaticExpose => duplicate_a_static_expose(agent), + Mutation::OverlapWithAnotherPeer => overlap_with_another_peer(agent), + }; + Some(bit) +} + +#[derive(Debug, Clone, Default)] +pub struct MutatedAgents(GatewayAgents); + +impl MutatedAgents { + #[must_use] + pub fn new(agents: GatewayAgents) -> Self { + Self(agents) + } +} + +impl ValueGenerator for MutatedAgents { + type Output = (Mutation, bool, GatewayAgent); + + fn generate(&self, d: &mut D) -> Option { + let mut agent = self.0.generate(d)?; + let all = Mutation::all(); + let mutation = all[d.gen_usize(Bound::Included(&0), Bound::Excluded(&all.len()))?]; + let bit = apply(d, &mut agent, mutation)?; + Some((mutation, bit, agent)) + } +} diff --git a/k8s-intf/src/bolero/peering.rs b/k8s-intf/src/bolero/peering.rs index cf7ad87af1..78bb62f7b9 100644 --- a/k8s-intf/src/bolero/peering.rs +++ b/k8s-intf/src/bolero/peering.rs @@ -6,8 +6,10 @@ use std::ops::Bound; use bolero::{Driver, ValueGenerator}; -use crate::bolero::expose::LegalValueExposeGenerator; -use crate::bolero::{SubnetMap, VpcSubnetMap}; +use crate::bolero::acl::{AclGenerator, SideFacts}; +use crate::bolero::expose::{ExposeGenerator, Which}; +use crate::bolero::support::blocks; +use crate::bolero::{AddressFamily, NatFlavour, SubnetMap, VpcSubnetMap}; use crate::gateway_agent_crd::{GatewayAgentPeerings, GatewayAgentPeeringsPeering}; /// Generate legal values for `GatewayAgentPeeringsPeering` @@ -16,12 +18,28 @@ use crate::gateway_agent_crd::{GatewayAgentPeerings, GatewayAgentPeeringsPeering /// In particular, subnet names are restricted. Lengths of various lists is also limited to 16 pub struct LegalValuePeeringsPeeringGenerator<'a> { subnets: &'a SubnetMap, + flavours: &'a [NatFlavour], + family: AddressFamily, + max_exposes: u8, + slot_base: u8, } impl<'a> LegalValuePeeringsPeeringGenerator<'a> { #[must_use] - pub fn new(subnets: &'a SubnetMap) -> Self { - Self { subnets } + pub fn new( + subnets: &'a SubnetMap, + flavours: &'a [NatFlavour], + family: AddressFamily, + max_exposes: u8, + vpc: u8, + ) -> Self { + Self { + subnets, + flavours, + family, + max_exposes, + slot_base: blocks::expose_slot(vpc, max_exposes, 0), + } } } @@ -29,11 +47,15 @@ impl ValueGenerator for LegalValuePeeringsPeeringGenerator<'_> { type Output = GatewayAgentPeeringsPeering; fn generate(&self, d: &mut D) -> Option { - let num_expose = d.gen_usize(Bound::Included(&1), Bound::Included(&16))?; - let expose_gen = LegalValueExposeGenerator::new(self.subnets); - let expose = (0..num_expose) - .map(|_| expose_gen.generate(d)) - .collect::>>()?; + let num_expose = d.gen_u8(Bound::Included(&1), Bound::Included(&self.max_exposes))?; + let mut expose = Vec::with_capacity(usize::from(num_expose)); + for index in 0..num_expose { + let flavour = self.flavours + [d.gen_usize(Bound::Included(&0), Bound::Excluded(&self.flavours.len()))?]; + let which = Which::nth(self.slot_base.saturating_add(index), index, num_expose); + expose + .push(ExposeGenerator::new(flavour, self.family, which, self.subnets).generate(d)?); + } Some(GatewayAgentPeeringsPeering { expose: Some(expose).filter(|e| !e.is_empty()), @@ -47,6 +69,10 @@ impl ValueGenerator for LegalValuePeeringsPeeringGenerator<'_> { pub struct LegalValuePeeringsGenerator<'a> { vpc_subnets: &'a VpcSubnetMap, vpc_names: Vec<&'a String>, + flavours: &'a [NatFlavour], + families: &'a [AddressFamily], + max_exposes: u8, + groups: &'a [String], } impl<'a> LegalValuePeeringsGenerator<'a> { @@ -55,46 +81,128 @@ impl<'a> LegalValuePeeringsGenerator<'a> { /// # Errors /// /// Returns an error if there are less than two VPCs in the subnet map. - pub fn new(vpc_subnets: &'a VpcSubnetMap) -> Result { + pub fn new( + vpc_subnets: &'a VpcSubnetMap, + flavours: &'a [NatFlavour], + families: &'a [AddressFamily], + max_exposes: u8, + groups: &'a [String], + ) -> Result { if vpc_subnets.len() < 2 { return Err("At least two VPCs are required to generate peerings".to_string()); } + if groups.is_empty() { + return Err("At least one gateway group is required".to_string()); + } let vpc_names = vpc_subnets.keys().collect(); Ok(Self { vpc_subnets, vpc_names, + flavours, + families, + max_exposes, + groups, }) } + + fn stateless_of(&self) -> Vec { + let stateless: Vec = self + .flavours + .iter() + .copied() + .filter(|flavour| !flavour.is_stateful()) + .collect(); + if stateless.is_empty() { + vec![NatFlavour::None] + } else { + stateless + } + } } -fn pick2<'a, D: Driver, T>(d: &mut D, items: &[&'a T]) -> Option<[&'a T; 2]> { - assert!(items.len() >= 2); +fn pick2(d: &mut D, len: usize) -> Option<[usize; 2]> { + assert!(len >= 2); - let index1 = d.gen_usize(Bound::Included(&0), Bound::Excluded(&items.len()))?; - let mut index2 = d.gen_usize(Bound::Included(&0), Bound::Excluded(&items.len()))?; + let index1 = d.gen_usize(Bound::Included(&0), Bound::Excluded(&len))?; + let mut index2 = d.gen_usize(Bound::Included(&0), Bound::Excluded(&len))?; if index1 == index2 { - index2 = (index2 + 1) % items.len(); + index2 = (index2 + 1) % len; } - Some([items[index1], items[index2]]) + Some([index1, index2]) } -impl ValueGenerator for LegalValuePeeringsGenerator<'_> { - type Output = GatewayAgentPeerings; +impl LegalValuePeeringsGenerator<'_> { + #[must_use] + pub fn pairs(&self) -> Vec<[usize; 2]> { + let n = self.vpc_names.len(); + let mut out = Vec::with_capacity(n * n / 2); + for first in 0..n { + for second in (first + 1)..n { + out.push([first, second]); + } + } + out + } + + pub fn generate_for( + &self, + d: &mut D, + vpcs: [usize; 2], + ) -> Option { + let vpc_names = [*self.vpc_names.get(vpcs[0])?, *self.vpc_names.get(vpcs[1])?]; + let family = self.families + [d.gen_usize(Bound::Included(&0), Bound::Excluded(&self.families.len()))?]; + + let stateful_side = d.gen_usize(Bound::Included(&0), Bound::Included(&1))?; + let stateless = self.stateless_of(); - fn generate(&self, d: &mut D) -> Option { - let vpc_names = pick2(d, &self.vpc_names)?; let empty_map = SubnetMap::new(); - let peerings_gens = vpc_names.map(|n| { - LegalValuePeeringsPeeringGenerator::new(self.vpc_subnets.get(n).unwrap_or(&empty_map)) - }); let peering = (0..=1) - .map(|i| Some((vpc_names[i].clone(), peerings_gens[i].generate(d)?))) + .map(|i| { + let flavours: &[NatFlavour] = if i == stateful_side { + self.flavours + } else { + &stateless + }; + let generator = LegalValuePeeringsPeeringGenerator::new( + self.vpc_subnets.get(vpc_names[i]).unwrap_or(&empty_map), + flavours, + family, + self.max_exposes, + u8::try_from(vpcs[i]).unwrap_or(u8::MAX), + ); + Some((vpc_names[i].clone(), generator.generate(d)?)) + }) .collect::>>()?; + let group = self.groups + [d.gen_usize(Bound::Included(&0), Bound::Excluded(&self.groups.len()))?] + .clone(); + + let acl = if d.produce::()? { + let facts: Vec = peering + .iter() + .map(|(vpc, manifest)| SideFacts::of(vpc, manifest)) + .collect(); + let [left, right] = <[SideFacts; 2]>::try_from(facts).ok()?; + Some(AclGenerator::new(left, right).generate(d)?) + } else { + None + }; + Some(GatewayAgentPeerings { - gateway_group: Some(d.produce::()?), + gateway_group: Some(group), peering: Some(peering), - acl: None, // FIXME: Add a proper implementation when used + acl, }) } } + +impl ValueGenerator for LegalValuePeeringsGenerator<'_> { + type Output = GatewayAgentPeerings; + + fn generate(&self, d: &mut D) -> Option { + let pair = pick2(d, self.vpc_names.len())?; + self.generate_for(d, pair) + } +} diff --git a/k8s-intf/src/bolero/permute.rs b/k8s-intf/src/bolero/permute.rs new file mode 100644 index 0000000000..779e411dd1 --- /dev/null +++ b/k8s-intf/src/bolero/permute.rs @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Open Network Fabric Authors + +use std::collections::BTreeMap; +use std::ops::Bound; + +use bolero::{Driver, ValueGenerator}; + +use crate::bolero::mutate::{MutatedAgents, Mutation}; +use crate::gateway_agent_crd::{ + GatewayAgent, GatewayAgentPeerings, GatewayAgentPeeringsPeeringExpose, +}; + +fn reorder(d: &mut D, items: &mut [T]) -> Option<()> { + if items.len() < 2 { + return Some(()); + } + let by = d.gen_usize(Bound::Included(&0), Bound::Excluded(&items.len()))?; + items.rotate_left(by); + let first = d.gen_usize(Bound::Included(&0), Bound::Excluded(&items.len()))?; + let second = d.gen_usize(Bound::Included(&0), Bound::Excluded(&items.len()))?; + items.swap(first, second); + Some(()) +} + +fn reorder_expose( + d: &mut D, + expose: &mut GatewayAgentPeeringsPeeringExpose, +) -> Option<()> { + if let Some(ips) = expose.ips.as_mut() { + reorder(d, ips)?; + } + if let Some(translations) = expose.r#as.as_mut() { + reorder(d, translations)?; + } + if let Some(ports) = expose + .nat + .as_mut() + .and_then(|nat| nat.port_forward.as_mut()) + .and_then(|forward| forward.ports.as_mut()) + { + reorder(d, ports)?; + } + Some(()) +} + +fn reorder_peering(d: &mut D, peering: &mut GatewayAgentPeerings) -> Option<()> { + for manifest in peering.peering.iter_mut().flatten().map(|(_, m)| m) { + if let Some(exposes) = manifest.expose.as_mut() { + reorder(d, exposes)?; + for expose in exposes.iter_mut() { + reorder_expose(d, expose)?; + } + } + } + + for rule in peering + .acl + .iter_mut() + .flat_map(|acl| acl.rules.iter_mut()) + .flatten() + { + let Some(pattern) = rule.r#match.as_mut() else { + continue; + }; + if let Some(src) = pattern.src.as_mut() { + reorder(d, src)?; + } + if let Some(dst) = pattern.dst.as_mut() { + reorder(d, dst)?; + } + } + Some(()) +} + +fn reorder_agent(d: &mut D, agent: &mut GatewayAgent) -> Option { + let before = format!("{:?}", agent.spec.peerings); + + if let Some(peerings) = agent.spec.peerings.as_mut() { + let names: Vec = peerings.keys().cloned().collect(); + let mut bodies: Vec = peerings.values().cloned().collect(); + reorder(d, &mut bodies)?; + *peerings = names.into_iter().zip(bodies).collect::>(); + + for peering in peerings.values_mut() { + reorder_peering(d, peering)?; + } + } + + Some(before != format!("{:?}", agent.spec.peerings)) +} + +#[derive(Debug, Default, Clone)] +pub struct PermutedAgents(MutatedAgents); + +impl PermutedAgents { + #[must_use] + pub fn new(agents: MutatedAgents) -> Self { + Self(agents) + } +} + +impl ValueGenerator for PermutedAgents { + type Output = (Mutation, GatewayAgent, GatewayAgent, bool); + + fn generate(&self, d: &mut D) -> Option { + let (mutation, _applied, agent) = self.0.generate(d)?; + let mut permuted = agent.clone(); + let moved = reorder_agent(d, &mut permuted)?; + Some((mutation, agent, permuted, moved)) + } +} diff --git a/k8s-intf/src/bolero/reduce.rs b/k8s-intf/src/bolero/reduce.rs new file mode 100644 index 0000000000..fca44ec1f9 --- /dev/null +++ b/k8s-intf/src/bolero/reduce.rs @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Open Network Fabric Authors + +use std::ops::Bound; + +use bolero::{Driver, ValueGenerator}; + +use crate::bolero::mutate::{MutatedAgents, Mutation}; +use crate::gateway_agent_crd::GatewayAgent; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Dropped { + pub peering: String, + pub vpc: String, + pub index: usize, + pub nat: Option<&'static str>, +} + +impl std::fmt::Display for Dropped { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}/{} expose {} ({})", + self.peering, + self.vpc, + self.index, + self.nat.unwrap_or("no nat") + ) + } +} + +fn candidates(agent: &GatewayAgent) -> Vec<(String, String, usize)> { + let mut out = Vec::new(); + for (peering_name, peering) in agent.spec.peerings.iter().flatten() { + for (vpc, manifest) in peering.peering.iter().flatten() { + let count = manifest.expose.as_ref().map_or(0, Vec::len); + if count < 2 { + continue; + } + for index in 0..count { + out.push((peering_name.clone(), vpc.clone(), index)); + } + } + } + out +} + +fn drop_an_expose( + d: &mut D, + agent: &mut GatewayAgent, + mut choices: Vec<(String, String, usize)>, +) -> Option { + let choice = d.gen_usize(Bound::Included(&0), Bound::Excluded(&choices.len()))?; + let (peering_name, vpc, index) = choices.swap_remove(choice); + + let exposes = agent + .spec + .peerings + .as_mut()? + .get_mut(&peering_name)? + .peering + .as_mut()? + .get_mut(&vpc)? + .expose + .as_mut()?; + if index >= exposes.len() { + return None; + } + let removed = exposes.remove(index); + let nat = removed.nat.as_ref().and_then(|nat| { + if nat.r#static.is_some() { + Some("static") + } else if nat.masquerade.is_some() { + Some("masquerade") + } else if nat.port_forward.is_some() { + Some("port forwarding") + } else { + None + } + }); + + Some(Dropped { + peering: peering_name, + vpc, + index, + nat, + }) +} + +#[derive(Debug, Default, Clone)] +pub struct ReducedAgents(MutatedAgents); + +impl ReducedAgents { + #[must_use] + pub fn new(agents: MutatedAgents) -> Self { + Self(agents) + } +} + +impl ValueGenerator for ReducedAgents { + type Output = (Mutation, GatewayAgent, GatewayAgent, Option); + + fn generate(&self, d: &mut D) -> Option { + let (mutation, _applied, agent) = self.0.generate(d)?; + let mut reduced = agent.clone(); + let choices = candidates(&reduced); + if choices.is_empty() { + return Some((mutation, agent, reduced, None)); + } + let dropped = drop_an_expose(d, &mut reduced, choices)?; + Some((mutation, agent, reduced, Some(dropped))) + } +} diff --git a/k8s-intf/src/bolero/spec.rs b/k8s-intf/src/bolero/spec.rs index e89abbda46..82e8beacec 100644 --- a/k8s-intf/src/bolero/spec.rs +++ b/k8s-intf/src/bolero/spec.rs @@ -9,7 +9,7 @@ use bolero::{Driver, TypeGenerator, ValueGenerator}; use lpm::prefix::Prefix; use crate::bolero::peering::LegalValuePeeringsGenerator; -use crate::bolero::{LegalValue, SubnetMap, VpcSubnetMap}; +use crate::bolero::{AddressFamily, LegalValue, NatFlavour, SubnetMap, VpcSubnetMap}; use crate::gateway_agent_crd::{ GatewayAgentGateway, GatewayAgentGroups, GatewayAgentSpec, GatewayAgentVpcs, }; @@ -44,6 +44,85 @@ fn increment_string(s: &mut str) { } } +#[derive(Debug, Clone)] +pub struct SpecBuilder { + max_vpcs: u8, + max_peerings: u8, + max_exposes: u8, + max_subnets: u8, + flavours: Vec, + families: Vec, +} + +impl Default for SpecBuilder { + fn default() -> Self { + Self { + max_vpcs: 4, + max_peerings: 3, + max_exposes: 2, + max_subnets: 3, + flavours: NatFlavour::all(), + families: AddressFamily::all(), + } + } +} + +impl SpecBuilder { + #[must_use] + pub fn max_vpcs(mut self, max: u8) -> Self { + self.max_vpcs = max; + self + } + + #[must_use] + pub fn max_peerings(mut self, max: u8) -> Self { + self.max_peerings = max; + self + } + + #[must_use] + pub fn max_exposes(mut self, max: u8) -> Self { + self.max_exposes = max; + self + } + + #[must_use] + pub fn max_subnets(mut self, max: u8) -> Self { + self.max_subnets = max; + self + } + + #[must_use] + pub fn flavours(mut self, flavours: Vec) -> Self { + if !flavours.is_empty() { + self.flavours = flavours; + } + self + } + + #[must_use] + pub fn families(mut self, families: Vec) -> Self { + if !families.is_empty() { + self.families = families; + } + self + } + + #[must_use] + pub fn build(self) -> GatewayAgentSpecs { + GatewayAgentSpecs(self) + } +} + +#[derive(Debug, Clone)] +pub struct GatewayAgentSpecs(pub(crate) SpecBuilder); + +impl Default for GatewayAgentSpecs { + fn default() -> Self { + SpecBuilder::default().build() + } +} + /// Generate a random legal `GatewayAgentSpec` /// /// This does not cover all legal `GatewayAgentSpecs`, @@ -52,9 +131,19 @@ fn increment_string(s: &mut str) { /// vni combinations are generated. impl TypeGenerator for LegalValue { fn generate(d: &mut D) -> Option { - let num_vpcs = d.gen_usize(Bound::Included(&0), Bound::Included(&16))?; + Some(LegalValue(GatewayAgentSpecs::default().generate(d)?)) + } +} + +impl ValueGenerator for GatewayAgentSpecs { + type Output = GatewayAgentSpec; + + fn generate(&self, d: &mut D) -> Option { + let knobs = &self.0; + let num_vpcs = + usize::from(d.gen_u8(Bound::Included(&0), Bound::Included(&knobs.max_vpcs))?); let num_peerings = if num_vpcs > 1 { - d.gen_usize(Bound::Included(&0), Bound::Included(&16))? + usize::from(d.gen_u8(Bound::Included(&0), Bound::Included(&knobs.max_peerings))?) } else { 0 }; @@ -64,8 +153,12 @@ impl TypeGenerator for LegalValue { let mut vpc_internal_ids = HashSet::new(); for i in 0..num_vpcs { let vni_offset = u32::try_from(i).expect("too many vpcs"); - let lv_vpc = d.produce::>()?; - let mut vpc = lv_vpc.take(); + let mut vpc = crate::bolero::vpc::VpcGenerator::new( + u8::try_from(i).unwrap_or(u8::MAX), + knobs.max_subnets, + &knobs.families, + ) + .generate(d)?; let vpc_id = vpc.internal_id.as_mut().unwrap(); while !vpc_internal_ids.insert(vpc_id.clone()) { // We already have a VPC with this internal_id, "increment" the string to generate a @@ -78,19 +171,31 @@ impl TypeGenerator for LegalValue { let vpc_subnet_map = extract_subnets(&vpcs); - let mut peerings = BTreeMap::new(); - if num_peerings > 0 { - let peering_gen = LegalValuePeeringsGenerator::new(&vpc_subnet_map).unwrap(); - for i in 0..num_peerings { - peerings.insert(format!("peering{i}"), peering_gen.generate(d)?); - } - } - let num_groups = d.gen_usize(Bound::Included(&0), Bound::Included(&6))?; let mut groups = BTreeMap::new(); for i in 0..=num_groups { groups.insert(format!("gwgroup-{i}"), d.produce::()?); } + let group_names: Vec = groups.keys().cloned().collect(); + + let mut peerings = BTreeMap::new(); + if num_peerings > 0 { + let peering_gen = LegalValuePeeringsGenerator::new( + &vpc_subnet_map, + &knobs.flavours, + &knobs.families, + knobs.max_exposes, + &group_names, + ) + .unwrap(); + let mut available = peering_gen.pairs(); + let wanted = num_peerings.min(available.len()); + for i in 0..wanted { + let choice = d.gen_usize(Bound::Included(&0), Bound::Excluded(&available.len()))?; + let pair = available.swap_remove(choice); + peerings.insert(format!("peering{i}"), peering_gen.generate_for(d, pair)?); + } + } let num_communities = d.gen_usize(Bound::Included(&0), Bound::Included(&9))?; let mut communities = BTreeMap::new(); @@ -99,7 +204,7 @@ impl TypeGenerator for LegalValue { communities.insert(i.to_string(), community); } - Some(LegalValue(GatewayAgentSpec { + Some(GatewayAgentSpec { agent_version: None, config: None, groups: Some(groups), @@ -107,6 +212,6 @@ impl TypeGenerator for LegalValue { gateway: Some(d.produce::>()?.take()), vpcs: Some(vpcs).filter(|v| !v.is_empty()), peerings: Some(peerings).filter(|p| !p.is_empty()), - })) + }) } } diff --git a/k8s-intf/src/bolero/support.rs b/k8s-intf/src/bolero/support.rs index b1880a381d..a6c2a28378 100644 --- a/k8s-intf/src/bolero/support.rs +++ b/k8s-intf/src/bolero/support.rs @@ -254,15 +254,22 @@ pub fn choose(d: &mut D, choices: &[T]) -> Option { Some(choices[index].clone()) } +const MIN_V4_MASK: u8 = 8; +const MIN_V6_MASK: u8 = 16; + pub fn generate_v4_prefixes(d: &mut D, count: u16) -> Option> { - let cidr4_gen = - UniqueV4CidrGenerator::new(count, d.gen_u8(Bound::Included(&0), Bound::Included(&32))?); + let cidr4_gen = UniqueV4CidrGenerator::new( + count, + d.gen_u8(Bound::Included(&MIN_V4_MASK), Bound::Included(&32))?, + ); cidr4_gen.generate(d) } pub fn generate_v6_prefixes(d: &mut D, count: u16) -> Option> { - let cidr6_gen = - UniqueV6CidrGenerator::new(count, d.gen_u8(Bound::Included(&0), Bound::Included(&128))?); + let cidr6_gen = UniqueV6CidrGenerator::new( + count, + d.gen_u8(Bound::Included(&MIN_V6_MASK), Bound::Included(&128))?, + ); cidr6_gen.generate(d) } @@ -416,3 +423,190 @@ mod test { } } } + +pub mod blocks { + use crate::bolero::AddressFamily; + use bolero::Driver; + use std::net::{Ipv4Addr, Ipv6Addr}; + + pub const SLOT_V4_LEN: u8 = 20; + pub const SLOT_V6_LEN: u8 = 48; + pub const MIN_V4_LEN: u8 = SLOT_V4_LEN; + pub const MIN_V6_LEN: u8 = SLOT_V6_LEN; + + pub const SUBNET_SLOTS: u8 = 16; + + #[must_use] + pub fn expose_slot(vpc: u8, slots_per_vpc: u8, expose: u8) -> u8 { + debug_assert!( + vpc < SUBNET_SLOTS, + "vpc {vpc} has no subnet slot of its own: {SUBNET_SLOTS} are reserved, so its subnets \ + would land on top of an expose's prefixes and the two would overlap" + ); + let wanted = u32::from(vpc) * u32::from(slots_per_vpc) + u32::from(expose); + debug_assert!( + u8::try_from(wanted).is_ok(), + "vpc {vpc} expose {expose} needs slot {wanted} of 256, so the saturating arithmetic \ + below will hand it a slot another expose already holds" + ); + vpc.saturating_mul(slots_per_vpc).saturating_add(expose) + } + + #[derive(Debug, Clone, Copy)] + pub struct At { + pub slot: u8, + pub sub: u8, + pub subs: u8, + } + + impl At { + #[must_use] + pub fn whole(slot: u8) -> Self { + Self { + slot, + sub: 0, + subs: 1, + } + } + + #[must_use] + pub fn nth(slot: u8, sub: u8, subs: u8) -> Self { + Self { + slot, + sub, + subs: subs.max(sub.saturating_add(1)), + } + } + + fn sub_bits(self) -> u8 { + u8::try_from(self.subs.max(1).next_power_of_two().trailing_zeros()).unwrap_or(0) + } + + fn level(self, family: AddressFamily) -> u8 { + min_len(family) + .saturating_add(self.sub_bits()) + .min(max_len(family)) + } + + fn place(self, family: AddressFamily, block_base: u128, slot_index: u32) -> (u128, u8) { + let width = max_len(family); + let level = self.level(family); + let slot = u128::from(slot_index) << (width - min_len(family)); + let sub_mask = (1u128 << self.sub_bits()) - 1; + let sub = (u128::from(self.sub) & sub_mask) << (width - level); + (block_base | slot | sub, level) + } + } + + #[must_use] + pub fn min_len_at(family: AddressFamily, at: At) -> u8 { + at.level(family) + } + + fn v4(base: u32, block_len: u8, host: u32, len: u8) -> String { + let block_host_bits = 32 - block_len; + let within = if block_host_bits >= 32 { + host + } else { + host & ((1u32 << block_host_bits) - 1) + }; + let mask = u32::MAX.checked_shl(u32::from(32 - len)).unwrap_or(0); + let addr = (base | within) & mask; + format!("{}/{len}", Ipv4Addr::from(addr)) + } + + fn v6(base: u128, block_len: u8, host: u128, len: u8) -> String { + let block_host_bits = 128 - block_len; + let within = if block_host_bits >= 128 { + host + } else { + host & ((1u128 << block_host_bits) - 1) + }; + let mask = u128::MAX.checked_shl(u32::from(128 - len)).unwrap_or(0); + let addr = (base | within) & mask; + format!("{}/{len}", Ipv6Addr::from(addr)) + } + + pub fn private(d: &mut D, family: AddressFamily, at: At, len: u8) -> Option { + let slot = u32::from(SUBNET_SLOTS) + u32::from(at.slot); + Some(if family.is_v4() { + let (base, level) = at.place(family, 0x0A00_0000, slot); + v4(u32::try_from(base).ok()?, level, d.produce::()?, len) + } else { + let (base, level) = at.place(family, 0x2001_0db8_0000_0000_0000_0000_0000_0000, slot); + v6(base, level, d.produce::()?, len) + }) + } + + pub fn public(d: &mut D, family: AddressFamily, at: At, len: u8) -> Option { + let slot = u32::from(at.slot); + Some(if family.is_v4() { + let (base, level) = at.place(family, 0xAC10_0000, slot); + v4(u32::try_from(base).ok()?, level, d.produce::()?, len) + } else { + let (base, level) = at.place(family, 0x2001_0db8_8000_0000_0000_0000_0000_0000, slot); + v6(base, level, d.produce::()?, len) + }) + } + + #[must_use] + pub fn min_subnet_len(family: AddressFamily, count: u16) -> u8 { + let bits = u8::try_from(count.max(1).next_power_of_two().trailing_zeros()).unwrap_or(0); + min_len(family).saturating_add(bits).min(max_len(family)) + } + + pub fn private_run( + d: &mut D, + family: AddressFamily, + vpc: u8, + len: u8, + count: u16, + ) -> Option> { + if count == 0 { + return Some(Vec::new()); + } + let slot_len = u32::from(min_len(family)); + let mut out = Vec::with_capacity(usize::from(count)); + if family.is_v4() { + let base = 0x0A00_0000 | (u32::from(vpc) << (32 - slot_len)); + let slots = 1u32 + .checked_shl(u32::from(len) - slot_len) + .unwrap_or(u32::MAX); + let first = d.produce::()? % slots; + let shift = u32::from(32 - len); + for i in 0..u32::from(count) { + let slot = (first + i) % slots; + let addr = base | slot.checked_shl(shift).unwrap_or(0); + out.push(format!("{}/{len}", Ipv4Addr::from(addr))); + } + } else { + let base = + 0x2001_0db8_0000_0000_0000_0000_0000_0000 | (u128::from(vpc) << (128 - slot_len)); + let slots = 1u128 + .checked_shl(u32::from(len) - slot_len) + .unwrap_or(u128::MAX); + let first = d.produce::()? % slots; + let shift = u32::from(128 - len); + for i in 0..u128::from(count) { + let slot = (first + i) % slots; + let addr = base | slot.checked_shl(shift).unwrap_or(0); + out.push(format!("{}/{len}", Ipv6Addr::from(addr))); + } + } + Some(out) + } + + #[must_use] + pub fn min_len(family: AddressFamily) -> u8 { + if family.is_v4() { + MIN_V4_LEN + } else { + MIN_V6_LEN + } + } + + #[must_use] + pub fn max_len(family: AddressFamily) -> u8 { + if family.is_v4() { 32 } else { 128 } + } +} diff --git a/k8s-intf/src/bolero/vpc.rs b/k8s-intf/src/bolero/vpc.rs index 0eb84b6b92..eaf5c74fd8 100644 --- a/k8s-intf/src/bolero/vpc.rs +++ b/k8s-intf/src/bolero/vpc.rs @@ -7,8 +7,8 @@ use bolero::{Driver, TypeGenerator, ValueGenerator}; use net::vxlan::Vni; -use crate::bolero::LegalValue; -use crate::bolero::support::{UniqueV4CidrGenerator, UniqueV6CidrGenerator}; +use crate::bolero::support::blocks; +use crate::bolero::{AddressFamily, LegalValue}; use crate::gateway_agent_crd::{GatewayAgentVpcs, GatewayAgentVpcsSubnets}; fn generate_internal_id(d: &mut D) -> Option { @@ -21,20 +21,59 @@ fn generate_internal_id(d: &mut D) -> Option { Some(result) } -impl TypeGenerator for LegalValue { - fn generate(d: &mut D) -> Option { +#[derive(Debug, Clone)] +pub struct VpcGenerator<'a> { + vpc: u8, + max_subnets: u8, + families: &'a [AddressFamily], +} + +impl<'a> VpcGenerator<'a> { + #[must_use] + pub fn new(vpc: u8, max_subnets: u8, families: &'a [AddressFamily]) -> Self { + Self { + vpc, + max_subnets, + families, + } + } + + fn wants(&self, family: AddressFamily) -> bool { + self.families.contains(&family) + } +} + +impl ValueGenerator for VpcGenerator<'_> { + type Output = GatewayAgentVpcs; + + fn generate(&self, d: &mut D) -> Option { let internal_id = generate_internal_id(d)?; let vni = d.produce::()?; - let v4_masklen = d.gen_u8(Bound::Included(&0), Bound::Included(&32))?; - let num_v4_cidrs = d.gen_u16(Bound::Included(&0), Bound::Included(&16))?; + let num_v4_cidrs = if self.wants(AddressFamily::V4) { + u16::from(d.gen_u8(Bound::Included(&0), Bound::Included(&self.max_subnets))?) + } else { + 0 + }; + let num_v6_cidrs = if self.wants(AddressFamily::V6) { + u16::from(d.gen_u8(Bound::Included(&0), Bound::Included(&self.max_subnets))?) + } else { + 0 + }; - let v6_masklen = d.gen_u8(Bound::Included(&0), Bound::Included(&128))?; - let num_v6_cidrs = d.gen_u16(Bound::Included(&0), Bound::Included(&16))?; - let v4_gen = UniqueV4CidrGenerator::new(num_v4_cidrs, v4_masklen); - let v6_gen = UniqueV6CidrGenerator::new(num_v6_cidrs, v6_masklen); + let v4_masklen = d.gen_u8( + Bound::Included(&blocks::min_subnet_len(AddressFamily::V4, num_v4_cidrs)), + Bound::Included(&32), + )?; + let v6_masklen = d.gen_u8( + Bound::Included(&blocks::min_subnet_len(AddressFamily::V6, num_v6_cidrs)), + Bound::Included(&128), + )?; - let subnets_cidrs = vec![v4_gen.generate(d)?, v6_gen.generate(d)?]; + let subnets_cidrs = vec![ + blocks::private_run(d, AddressFamily::V4, self.vpc, v4_masklen, num_v4_cidrs)?, + blocks::private_run(d, AddressFamily::V6, self.vpc, v6_masklen, num_v6_cidrs)?, + ]; let subnets = subnets_cidrs .into_iter() .flatten() @@ -47,10 +86,17 @@ impl TypeGenerator for LegalValue { }) .collect::>(); - Some(LegalValue(GatewayAgentVpcs { + Some(GatewayAgentVpcs { internal_id: Some(internal_id), vni: Some(vni.into()), subnets: Some(subnets).filter(|s| !s.is_empty()), - })) + }) + } +} + +impl TypeGenerator for LegalValue { + fn generate(d: &mut D) -> Option { + let families = AddressFamily::all(); + Some(LegalValue(VpcGenerator::new(0, 3, &families).generate(d)?)) } } diff --git a/mgmt/Cargo.toml b/mgmt/Cargo.toml index 482e1a1396..366466fa0f 100644 --- a/mgmt/Cargo.toml +++ b/mgmt/Cargo.toml @@ -55,11 +55,16 @@ tokio = { workspace = true, features = ["macros", "rt", "sync", "time"] } tracing = { workspace = true, features = ["attributes"] } tracing-test = { workspace = true } +[lints.rust] +unexpected_cfgs = { level = "warn", check-cfg = ['cfg(fuzzing)'] } + [dev-dependencies] # internal dpdk = { workspace = true, features = ["test"] } # EAL for tests that build the rte_acl-backed ACL filter and flow-filter context fixin = { workspace = true } id = { workspace = true, features = ["bolero"] } +config = { workspace = true, features = ["bolero"] } +k8s-intf = { workspace = true, features = ["bolero"] } interface-manager = { workspace = true, features = ["bolero"] } lpm = { workspace = true, features = ["testing"] } n-vm = { workspace = true } diff --git a/mgmt/src/processor/confbuild/internal.rs b/mgmt/src/processor/confbuild/internal.rs index fc108e4871..a541059180 100644 --- a/mgmt/src/processor/confbuild/internal.rs +++ b/mgmt/src/processor/confbuild/internal.rs @@ -15,7 +15,7 @@ use config::external::overlay::vpc::{ValidatedPeering, ValidatedVpc}; use config::external::overlay::vpcpeering::ValidatedManifest; use config::{ConfigError, ConfigResult}; -use lpm::prefix::Prefix; +use lpm::prefix::{Prefix, PrefixWithOptionalPorts}; use net::route::RouteTableId; use net::vxlan::Vni; use std::net::Ipv4Addr; @@ -46,24 +46,30 @@ fn vpc_import_prefix_list_for_peer( Some(vpc.import_plist_peer_desc(rmanifest.name())), ); for expose in rmanifest.valexp() { + reject_ipv6(expose.ips().iter().map(PrefixWithOptionalPorts::prefix))?; // allow native prefixes, natted or not - let native_prefixes = - expose - .ips() - .iter() - .filter(|p| p.prefix().is_ipv4()) - .map(|prefix_with_ports| { - PrefixListEntry::new( - PrefixListAction::Permit, - PrefixListPrefix::Prefix(prefix_with_ports.prefix()), - Some(PrefixListMatchLen::Ge(prefix_with_ports.prefix().length())), - ) - }); + let native_prefixes = expose.ips().iter().map(|prefix_with_ports| { + PrefixListEntry::new( + PrefixListAction::Permit, + PrefixListPrefix::Prefix(prefix_with_ports.prefix()), + Some(PrefixListMatchLen::Ge(prefix_with_ports.prefix().length())), + ) + }); plist.add_entries(native_prefixes)?; } Ok(plist) } +fn reject_ipv6(prefixes: impl IntoIterator) -> ConfigResult { + if prefixes.into_iter().any(|prefix| prefix.is_ipv6()) { + return Err(ConfigError::Unsupported( + "IPv6 prefixes in a vpc peering: the FRR configuration built from a peering is \ + IPv4-only, so such a peering cannot be rendered", + )); + } + Ok(()) +} + /// Build AF l2vpn EVPN config for a VPC VRF #[must_use] fn vpc_bgp_af_l2vpn_evpn(vpc: &ValidatedVpc) -> AfL2vpnEvpn { @@ -156,6 +162,8 @@ impl VpcRoutingConfigIpv4 { nets.sort_unstable(); nets.dedup(); + reject_ipv6(nets.iter().copied())?; + /* list of advertised prefixes */ self.adv_nets.extend(nets.clone()); @@ -311,7 +319,7 @@ fn build_internal_overlay_config( Ok(()) } -const EVPN_RMAP_NO_ADV_COMM: &str = "EVPN-ROUTE-MAP-NO-ADV-COMM"; +pub(crate) const EVPN_RMAP_NO_ADV_COMM: &str = "EVPN-ROUTE-MAP-NO-ADV-COMM"; /// Create a route-map that adds community "no-advertise" to all routes fn route_map_add_noadv_comm() -> RouteMap { @@ -392,3 +400,165 @@ pub fn build_internal_config( debug!("Successfully built internal config for genid {genid}"); Ok(internal) } + +#[cfg(test)] +mod chain_properties { + use super::*; + use config::{ExternalConfig, GenId}; + use k8s_intf::bolero::AddressFamily; + use k8s_intf::bolero::crd::{GatewayAgentBuilder, GatewayAgents}; + use k8s_intf::gateway_agent_crd::GatewayAgent; + use routing::Render; + use std::collections::BTreeSet; + + fn ipv4_agents() -> GatewayAgents { + GatewayAgentBuilder::new() + .families(vec![AddressFamily::V4]) + .build() + } + + fn chain(agent: &GatewayAgent) -> Option<(GenId, InternalConfig)> { + let external = ExternalConfig::try_from(agent) + .unwrap_or_else(|e| panic!("a schema-legal CRD did not convert: {e}")); + let validated = external.validate().ok()?; + let genid = validated.genid(); + Some((genid, build_or_skip(&validated)?)) + } + + fn build_or_skip(validated: &ValidatedGwConfig) -> Option { + match build_internal_config(validated, None) { + Ok(internal) => Some(internal), + Err(ConfigError::Unsupported(_)) => None, + Err(e) => panic!("a validated configuration would not build: {e}\n{validated:#?}"), + } + } + + #[test] + fn whatever_validates_builds_and_renders() { + bolero::check!() + .with_generator(ipv4_agents()) + .for_each(|agent| { + let Some((genid, internal)) = chain(agent) else { + return; + }; + let text = internal.render(&genid).to_string(); + assert!( + text.contains(&format!("! config for gen {genid}")), + "the rendered config does not say which generation it is for" + ); + }); + } + + #[test] + fn every_vpc_gets_a_vrf_and_no_more() { + bolero::check!() + .with_generator(ipv4_agents()) + .for_each(|agent| { + let external = ExternalConfig::try_from(agent) + .unwrap_or_else(|e| panic!("a schema-legal CRD did not convert: {e}")); + let Ok(validated) = external.validate() else { + return; + }; + let Some(internal) = build_or_skip(&validated) else { + return; + }; + + let wanted: BTreeSet = validated + .external() + .overlay() + .vpc_table() + .values() + .map(|vpc| vpc.vni().as_u32()) + .collect(); + let built: BTreeSet = internal + .vrfs + .iter_by_name() + .filter_map(|vrf| vrf.vni.map(|vni| vni.as_u32())) + .collect(); + assert_eq!(built, wanted, "vrfs do not match the vpcs they come from"); + }); + } + + #[test] + fn the_properties_are_not_vacuous() { + use concurrency::sync::atomic::{AtomicUsize, Ordering}; + static SEEN: AtomicUsize = AtomicUsize::new(0); + static VALIDATED: AtomicUsize = AtomicUsize::new(0); + static VPCS: AtomicUsize = AtomicUsize::new(0); + static PEERINGS: AtomicUsize = AtomicUsize::new(0); + static ACLS: AtomicUsize = AtomicUsize::new(0); + + bolero::check!() + .with_generator(ipv4_agents()) + .for_each(|agent| { + SEEN.fetch_add(1, Ordering::Relaxed); + let external = ExternalConfig::try_from(agent) + .unwrap_or_else(|e| panic!("a schema-legal CRD did not convert: {e}")); + if let Ok(validated) = external.validate() { + VALIDATED.fetch_add(1, Ordering::Relaxed); + let table = validated.external().overlay().vpc_table(); + VPCS.fetch_add(table.len(), Ordering::Relaxed); + PEERINGS.fetch_add( + table + .values() + .map(|vpc| vpc.peerings().len()) + .sum::(), + Ordering::Relaxed, + ); + ACLS.fetch_add( + table + .values() + .flat_map(|vpc| vpc.peerings()) + .filter(|peering| peering.acl().is_some()) + .count(), + Ordering::Relaxed, + ); + } + }); + + let seen = SEEN.load(Ordering::Relaxed); + let validated = VALIDATED.load(Ordering::Relaxed); + let vpcs = VPCS.load(Ordering::Relaxed); + let peerings = PEERINGS.load(Ordering::Relaxed); + let acls = ACLS.load(Ordering::Relaxed); + println!( + "{validated}/{seen} validated, carrying {vpcs} vpcs, {peerings} peerings, {acls} acls" + ); + assert!(seen > 0, "no configurations were generated"); + assert!( + validated * 2 >= seen, + "only {validated} of {seen} configurations validated: the properties above are \ + checking much less than they look like they are" + ); + assert!(vpcs > validated, "validated configurations carry no vpcs"); + assert!( + peerings > 0, + "no validated configuration carries a peering, so nothing downstream of validation \ + has seen the exposes or the NAT" + ); + assert!( + acls * 2 >= validated, + "only {acls} of {validated} validated configurations carry an ACL: most generated ACLs \ + are being refused for something other than what they say" + ); + } + + #[test] + fn the_chain_is_deterministic() { + bolero::check!() + .with_generator(ipv4_agents()) + .for_each(|agent| { + let Some((genid, once)) = chain(agent) else { + return; + }; + let (_, twice) = chain(agent).unwrap_or_else(|| { + panic!("the same CRD validated once and not the second time") + }); + assert_eq!( + once.render(&genid).to_string(), + twice.render(&genid).to_string(), + "the configuration chain is not deterministic" + ); + }); + } +} diff --git a/mgmt/src/tests/mgmt.rs b/mgmt/src/tests/mgmt.rs index 3aa26792a1..98e39671a0 100644 --- a/mgmt/src/tests/mgmt.rs +++ b/mgmt/src/tests/mgmt.rs @@ -171,7 +171,7 @@ pub mod test { } /* DEVICE configuration */ - fn sample_device_config() -> DeviceConfig { + pub(super) fn sample_device_config() -> DeviceConfig { DeviceConfig::new() } @@ -319,7 +319,7 @@ pub mod test { } /* build sample underlay config */ - fn sample_underlay_config() -> Underlay { + pub(super) fn sample_underlay_config() -> Underlay { /* main loopback for BGP and vtep */ let loopback = IpAddr::from_str("7.0.0.100").expect("Bad address"); let router_id = get_v4_addr(loopback); @@ -333,7 +333,7 @@ pub mod test { } #[rustfmt::skip] - fn sample_gw_groups() -> GwGroupTable { + pub(super) fn sample_gw_groups() -> GwGroupTable { let mut gwt = GwGroupTable::new(); let mut group = GwGroup::new("gw-group-1"); group.add_member(GwGroupMember::new("gw1", 1, IpAddr::from_str("172.128.0.1").unwrap())).unwrap(); @@ -348,7 +348,7 @@ pub mod test { gwt } - fn sample_community_table() -> PriorityCommunityTable { + pub(super) fn sample_community_table() -> PriorityCommunityTable { let mut comtable = PriorityCommunityTable::new(); comtable.insert(0, "65000:800").unwrap(); comtable.insert(1, "65000:801").unwrap(); @@ -508,3 +508,736 @@ pub mod test { router.stop(); } } + +#[cfg(test)] +mod peering_chain { + use bolero::{Driver, ValueGenerator}; + use config::ExternalConfig; + use config::external::ExternalConfigBuilder; + use config::external::gwgroup::{GwGroup, GwGroupMember, GwGroupTable}; + use config::external::overlay::vpcpeering::VpcExpose; + use config::external::overlay::vpcpeering::contract::{ + LOCAL_VNI, MasqueradeExpose, PortForwardingExpose, REMOTE_VNI, StaticNatExpose, + overlay_with_exposes, + }; + use routing::Render; + use std::net::IpAddr; + use std::ops::Bound::Included; + use std::str::FromStr; + + use super::test::{ + sample_community_table, sample_device_config, sample_gw_groups, sample_underlay_config, + }; + use crate::processor::confbuild::internal::{EVPN_RMAP_NO_ADV_COMM, build_internal_config}; + + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + enum Flavour { + PortForwarding, + Masquerade, + Static, + } + + const MAX_EXPOSES: u8 = 3; + + const UNDERLAY_ASN: u32 = 65000; + + #[derive(Debug, Clone, Copy, Default)] + struct AnyNatExposes; + + impl ValueGenerator for AnyNatExposes { + type Output = Vec<(Flavour, VpcExpose)>; + + fn generate(&self, driver: &mut D) -> Option { + let count = driver.gen_u8(Included(&1), Included(&MAX_EXPOSES))?; + let mut out = Vec::with_capacity(usize::from(count)); + for _ in 0..count { + out.push(match driver.gen_u8(Included(&0), Included(&2))? { + 0 => ( + Flavour::PortForwarding, + PortForwardingExpose.generate(driver)?, + ), + 1 => (Flavour::Masquerade, MasqueradeExpose.generate(driver)?), + _ => (Flavour::Static, StaticNatExpose.generate(driver)?), + }); + } + Some(out) + } + } + + fn gw_groups_with_default() -> GwGroupTable { + let mut groups = sample_gw_groups(); + let mut default = GwGroup::new("default"); + default + .add_member(GwGroupMember::new( + "gw-default", + 1, + IpAddr::from_str("172.128.0.9").unwrap_or_else(|_| unreachable!()), + )) + .unwrap_or_else(|e| unreachable!("{e}")); + groups + .add_group(default) + .unwrap_or_else(|e| unreachable!("{e}")); + groups + } + + fn external_offering(exposes: Vec) -> ExternalConfig { + let overlay = overlay_with_exposes(exposes).unwrap_or_else(|e| unreachable!("{e}")); + ExternalConfigBuilder::default() + .gwname("test-gw".to_string()) + .genid(1) + .device(sample_device_config()) + .underlay(sample_underlay_config()) + .overlay(overlay) + .gwgroups(gw_groups_with_default()) + .communities(sample_community_table()) + .build() + .unwrap_or_else(|e| unreachable!("{e}")) + } + + #[test] + fn a_config_with_a_nat_peering_builds_and_renders() { + use concurrency::sync::atomic::{AtomicUsize, Ordering}; + static SEEN: AtomicUsize = AtomicUsize::new(0); + static BUILT: AtomicUsize = AtomicUsize::new(0); + static MULTI: AtomicUsize = AtomicUsize::new(0); + + bolero::check!() + .with_generator(AnyNatExposes) + .cloned() + .for_each(|offered: Vec<(Flavour, VpcExpose)>| { + SEEN.fetch_add(1, Ordering::Relaxed); + let flavours: Vec = offered.iter().map(|(f, _)| *f).collect(); + let exposes: Vec = offered.iter().map(|(_, e)| e.clone()).collect(); + let external = external_offering(exposes.clone()); + + let validated = match external.validate() { + Ok(validated) => validated, + Err(e) => { + assert!( + exposes.len() > 1, + "a single {:?} expose that is valid by construction was refused: {e}\n{exposes:#?}", + flavours[0] + ); + return; + } + }; + BUILT.fetch_add(1, Ordering::Relaxed); + if exposes.len() > 1 { + MULTI.fetch_add(1, Ordering::Relaxed); + } + + let internal = build_internal_config(&validated, None).unwrap_or_else(|e| { + panic!("a validated {flavours:?} configuration would not build: {e}\n{exposes:#?}") + }); + + let vnis: Vec = internal + .vrfs + .iter_by_name() + .filter_map(|vrf| vrf.vni.map(|vni| vni.as_u32())) + .collect(); + for vni in [LOCAL_VNI, REMOTE_VNI] { + assert!( + vnis.contains(&vni), + "vni {vni} of the peering is missing from the built config, got {vnis:?}" + ); + } + + assert!( + internal.vrfs.default_vrf_config().is_some(), + "the built config has no default vrf" + ); + + for order in 0..5 { + assert_eq!( + internal.commtable.get_community(order), + validated.external().communities().get_community(order), + "community {order} did not survive the build" + ); + } + + assert!( + internal + .rmap_table + .values() + .any(|rmap| rmap.name == EVPN_RMAP_NO_ADV_COMM), + "the evpn no-advertise route-map is missing from the built config" + ); + + let text = internal.render(&validated.genid()).to_string(); + assert!( + text.contains("! config for gen 1"), + "the rendered config does not say which generation it is for" + ); + assert!( + text.contains(EVPN_RMAP_NO_ADV_COMM), + "the evpn no-advertise route-map is missing from the rendered config" + ); + assert!( + text.contains(&format!("router bgp {UNDERLAY_ASN}")), + "the underlay bgp instance is missing from the rendered config" + ); + for vni in [LOCAL_VNI, REMOTE_VNI] { + assert!( + text.contains(&format!(" vni {vni}")), + "vni {vni} is missing from the rendered config" + ); + } + }); + + let seen = SEEN.load(Ordering::Relaxed); + let built = BUILT.load(Ordering::Relaxed); + let multi = MULTI.load(Ordering::Relaxed); + println!("{built}/{seen} configurations built, {multi} of them with several exposes"); + assert!( + built * 2 >= seen, + "most configurations were skipped: {built}/{seen}" + ); + assert!( + multi > 0, + "no configuration with more than one expose was built" + ); + } +} + +#[cfg(test)] +mod dataplane_tables { + use config::{ExternalConfig, ValidatedGwConfig}; + use flow_entry::flow_table::FlowTable; + use k8s_intf::bolero::NatFlavour; + use k8s_intf::bolero::crd::GatewayAgentBuilder; + use nat::masquerade::{MasqueradeConfig, NatAllocatorWriter}; + use nat::portfw::{PortFwTableWriter, build_port_forwarding_configuration}; + use nat::static_nat::NatTablesWriter; + use nat::static_nat::setup::build_nat_configuration; + + fn build_tables(validated: &ValidatedGwConfig, flavour: NatFlavour) { + let vpc_table = validated.external().overlay().vpc_table(); + + let nat_tables = build_nat_configuration(vpc_table).unwrap_or_else(|e| { + panic!("a validated {flavour:?} configuration would not build static NAT: {e}") + }); + let mut nattablesw = NatTablesWriter::new(); + nattablesw.update_nat_tables(nat_tables); + + let masquerade = MasqueradeConfig::new(vpc_table).set_randomize(false); + let mut natallocatorw = NatAllocatorWriter::new(); + let flow_table = FlowTable::new(16); + natallocatorw.update_nat_allocator(masquerade, validated.genid(), &flow_table); + + let ruleset = build_port_forwarding_configuration(vpc_table).unwrap_or_else(|e| { + panic!("a validated {flavour:?} configuration would not build port forwarding: {e}") + }); + let mut portfw_w = PortFwTableWriter::new(); + portfw_w.update_table(&ruleset).unwrap_or_else(|e| { + panic!("a validated {flavour:?} port-forwarding ruleset was refused by the table: {e}") + }); + } + + fn drive(flavour: NatFlavour) { + use concurrency::sync::atomic::{AtomicUsize, Ordering}; + let seen = AtomicUsize::new(0); + let built = AtomicUsize::new(0); + + let generator = GatewayAgentBuilder::new().flavours(vec![flavour]).build(); + + bolero::check!() + .with_generator(generator) + .cloned() + .for_each(|agent| { + seen.fetch_add(1, Ordering::Relaxed); + let external = ExternalConfig::try_from(&agent) + .unwrap_or_else(|e| panic!("a schema-legal CRD did not convert: {e}")); + let Ok(validated) = external.validate() else { + return; + }; + built.fetch_add(1, Ordering::Relaxed); + build_tables(&validated, flavour); + }); + + let seen = seen.load(Ordering::Relaxed); + let built = built.load(Ordering::Relaxed); + println!("{flavour:?}: {built}/{seen} configurations validated and built their tables"); + assert!( + built * 2 >= seen, + "only {built} of {seen} {flavour:?} configurations validated, so this checked much \ + less than it looks like it did" + ); + } + + #[test] + fn a_static_nat_configuration_builds_its_tables() { + drive(NatFlavour::Static); + } + + #[test] + fn a_masquerade_configuration_builds_its_tables() { + drive(NatFlavour::Masquerade); + } + + #[test] + fn a_port_forwarding_configuration_builds_its_tables() { + drive(NatFlavour::PortForward); + } + + #[test] + fn a_configuration_with_no_nat_builds_its_tables() { + drive(NatFlavour::None); + } +} + +#[cfg(test)] +#[cfg(test)] +mod enacted { + use config::{ConfigError, ExternalConfig, ValidatedGwConfig}; + use flow_entry::flow_table::FlowTable; + use k8s_intf::gateway_agent_crd::GatewayAgent; + use nat::masquerade::{MasqueradeConfig, NatAllocatorWriter}; + use nat::portfw::{PortFwTableWriter, build_port_forwarding_configuration}; + use nat::static_nat::setup::build_nat_configuration; + use routing::Render; + + use crate::processor::confbuild::internal::build_internal_config; + + pub(super) fn validator(crd: &GatewayAgent) -> Result { + let external = ExternalConfig::try_from(crd) + .map_err(|e| ConfigError::Invalid(format!("conversion: {e}")))?; + external.validate() + } + + pub(super) struct Artifacts { + pub frr: Vec, + pub static_nat: Vec, + pub port_forwarding: Vec, + pub masquerade: Vec, + } + + fn lines(text: &str) -> Vec { + let mut out: Vec = text + .lines() + .map(|line| line.trim_end().to_string()) + .filter(|line| !line.is_empty()) + .collect(); + out.sort(); + out + } + + impl Artifacts { + pub(super) fn of(validated: &ValidatedGwConfig) -> Option { + let genid = validated.genid(); + let internal = build_internal_config(validated, None).ok()?; + let vpc_table = validated.external().overlay().vpc_table(); + + let nat_tables = build_nat_configuration(vpc_table).ok()?; + let ruleset = build_port_forwarding_configuration(vpc_table).ok()?; + let mut portfw = PortFwTableWriter::new(); + portfw.update_table(&ruleset).ok()?; + + let masquerade = MasqueradeConfig::new(vpc_table).set_randomize(false); + let mut writer = NatAllocatorWriter::new(); + writer.update_nat_allocator(masquerade, genid, &FlowTable::new(16)); + let allocator = writer.get_reader().get(); + + Some(Self { + frr: lines(&internal.render(&genid).to_string()), + static_nat: lines(&nat_tables.to_string()), + port_forwarding: lines( + &ruleset + .iter() + .map(ToString::to_string) + .collect::>() + .join("\n"), + ), + masquerade: allocator + .map(|allocator| lines(&allocator.to_string())) + .unwrap_or_default(), + }) + } + + pub(super) fn all(&self) -> Vec { + let mut out = self.frr.clone(); + out.extend(self.static_nat.iter().cloned()); + out.extend(self.port_forwarding.iter().cloned()); + out.extend(self.masquerade.iter().cloned()); + out.sort(); + out + } + } +} + +mod validator_completeness { + use super::enacted::validator; + use concurrency::sync::atomic::{AtomicUsize, Ordering}; + use config::{ConfigError, ValidatedGwConfig}; + use flow_entry::flow_table::FlowTable; + use k8s_intf::bolero::AddressFamily; + use k8s_intf::bolero::crd::GatewayAgentBuilder; + use k8s_intf::bolero::mutate::{MutatedAgents, Mutation}; + use k8s_intf::gateway_agent_crd::GatewayAgent; + use nat::masquerade::{MasqueradeConfig, NatAllocatorWriter}; + use nat::portfw::{PortFwTableWriter, build_port_forwarding_configuration}; + use nat::static_nat::NatTablesWriter; + use nat::static_nat::setup::build_nat_configuration; + use routing::Render; + + use crate::processor::confbuild::internal::build_internal_config; + + fn enact(validated: &ValidatedGwConfig, mutation: Mutation) { + let genid = validated.genid(); + + let internal = build_internal_config(validated, None).unwrap_or_else(|e| { + panic!("{mutation:?}: validator accepted a config the builder rejects: {e}") + }); + let _ = internal.render(&genid).to_string(); + + let vpc_table = validated.external().overlay().vpc_table(); + + let nat_tables = build_nat_configuration(vpc_table).unwrap_or_else(|e| { + panic!("{mutation:?}: validator accepted a config static NAT rejects: {e}") + }); + NatTablesWriter::new().update_nat_tables(nat_tables); + + let masquerade = MasqueradeConfig::new(vpc_table).set_randomize(false); + NatAllocatorWriter::new().update_nat_allocator(masquerade, genid, &FlowTable::new(16)); + + let ruleset = build_port_forwarding_configuration(vpc_table).unwrap_or_else(|e| { + panic!("{mutation:?}: validator accepted a config port forwarding rejects: {e}") + }); + PortFwTableWriter::new() + .update_table(&ruleset) + .unwrap_or_else(|e| { + panic!("{mutation:?}: validator accepted a ruleset the table rejects: {e}") + }); + } + + #[test] + fn whatever_the_validator_accepts_can_be_enacted_over_ipv4() { + let families = vec![AddressFamily::V4]; + + const N: usize = Mutation::COUNT; + #[allow(clippy::declare_interior_mutable_const)] + const ZERO: AtomicUsize = AtomicUsize::new(0); + static DRAWN: [AtomicUsize; N] = [ZERO; N]; + static APPLIED: [AtomicUsize; N] = [ZERO; N]; + static REFUSED: [AtomicUsize; N] = [ZERO; N]; + + bolero::check!() + .with_generator(MutatedAgents::new( + GatewayAgentBuilder::new().families(families).build(), + )) + .cloned() + .for_each(|(mutation, bit, agent): (Mutation, bool, GatewayAgent)| { + let outcome = validator(&agent); + let accepted = outcome.is_ok(); + + if let Ok(validated) = &outcome { + assert!( + !bit, + "{mutation:?} broke a rule outright and the validator accepted the result, \ + so nothing downstream will ever report it" + ); + enact(validated, mutation); + } else if let Err(e) = &outcome { + assert!( + mutation != Mutation::None, + "an unmutated configuration was refused, so the generator is producing \ + illegal input and every mutated case is suspect: {e}" + ); + + assert!( + !matches!(e, ConfigError::InternalFailure(_)), + "{mutation:?}: rejected with an internal failure, which tells the user \ + nothing they can act on: {e}" + ); + } + + let slot = mutation.index(); + DRAWN[slot].fetch_add(1, Ordering::Relaxed); + if bit { + APPLIED[slot].fetch_add(1, Ordering::Relaxed); + } + if !accepted { + REFUSED[slot].fetch_add(1, Ordering::Relaxed); + } + }); + + #[cfg(fuzzing)] + println!("under a coverage-guided engine: skipping the generator-health checks"); + + #[cfg(not(fuzzing))] + check_generator_health(&DRAWN, &APPLIED, &REFUSED); + } + + #[cfg(not(fuzzing))] + fn check_generator_health( + drawn_at: &[AtomicUsize; Mutation::COUNT], + applied_at: &[AtomicUsize; Mutation::COUNT], + refused_at: &[AtomicUsize; Mutation::COUNT], + ) { + let mut total = 0; + for mutation in Mutation::all() { + let slot = mutation.index(); + let drawn = drawn_at[slot].load(Ordering::Relaxed); + let applied = applied_at[slot].load(Ordering::Relaxed); + let refused = refused_at[slot].load(Ordering::Relaxed); + println!("{mutation:<32?} {drawn:>7} drawn {applied:>7} applied {refused:>7} refused"); + total += drawn; + } + + let cases_for_drawn = 50 * Mutation::COUNT; + let cases_for_applied = 2_000 * Mutation::COUNT; + if total < cases_for_drawn { + println!( + "only {total} cases: too few to say anything about mutation coverage \ + (needs {cases_for_drawn} to check each was drawn, {cases_for_applied} to check each \ + found a target)" + ); + return; + } + + for mutation in Mutation::all() { + assert!( + drawn_at[mutation.index()].load(Ordering::Relaxed) > 0, + "{mutation:?} was never drawn in {total} cases" + ); + } + + if total < cases_for_applied { + println!( + "{total} cases: enough to check every mutation was drawn, too few to check each \ + found a target (needs {cases_for_applied})" + ); + return; + } + + for mutation in Mutation::all().into_iter().filter(|m| *m != Mutation::None) { + assert!( + applied_at[mutation.index()].load(Ordering::Relaxed) > 0, + "{mutation:?} never found anything to break in {total} cases, so it is testing \ + nothing" + ); + } + } +} + +mod ambiguity { + use super::enacted::{Artifacts, validator}; + use concurrency::sync::atomic::{AtomicUsize, Ordering}; + use k8s_intf::bolero::mutate::Mutation; + use k8s_intf::bolero::permute::PermutedAgents; + use k8s_intf::gateway_agent_crd::GatewayAgent; + + #[test] + fn a_configuration_has_only_one_meaning() { + static MOVED: AtomicUsize = AtomicUsize::new(0); + static COMPARED: AtomicUsize = AtomicUsize::new(0); + + bolero::check!() + .with_generator(PermutedAgents::new( + k8s_intf::bolero::mutate::MutatedAgents::new( + k8s_intf::bolero::crd::GatewayAgentBuilder::new() + .families(vec![k8s_intf::bolero::AddressFamily::V4]) + .build(), + ), + )) + .cloned() + .for_each( + |(mutation, agent, permuted, moved): (Mutation, GatewayAgent, GatewayAgent, bool)| { + let Ok(first) = validator(&agent) else { + return; + }; + + let second = validator(&permuted).unwrap_or_else(|e| { + panic!( + "{mutation:?}: reordering a configuration's sets made the validator refuse \ + it, so it is treating list order as meaning: {e}" + ) + }); + + let (Some(before), Some(after)) = (Artifacts::of(&first), Artifacts::of(&second)) + else { + return; + }; + let (before, after) = (before.all(), after.all()); + + if moved { + MOVED.fetch_add(1, Ordering::Relaxed); + } + COMPARED.fetch_add(1, Ordering::Relaxed); + + if before != after { + let mut differences: Vec = Vec::new(); + for line in &before { + if !after.contains(line) { + differences.push(format!(" only before: {line}")); + } + } + for line in &after { + if !before.contains(line) { + differences.push(format!(" only after: {line}")); + } + } + differences.truncate(20); + panic!( + "{mutation:?}: reordering a configuration's sets changed what the dataplane \ + installs, so the configuration had more than one meaning and the chain \ + picked one:\n{}", + differences.join("\n") + ); + } + }, + ); + + let compared = COMPARED.load(Ordering::Relaxed); + let moved = MOVED.load(Ordering::Relaxed); + println!("{moved} of {compared} comparisons were of a genuinely reordered configuration"); + #[cfg(not(fuzzing))] + assert!( + compared > 0 && moved * 10 >= compared, + "only {moved} of {compared} comparisons actually reordered anything: the permutation is \ + not doing any work" + ); + } +} + +mod relevance { + use super::enacted::{Artifacts, validator}; + use concurrency::sync::atomic::{AtomicUsize, Ordering}; + use k8s_intf::bolero::mutate::Mutation; + use k8s_intf::bolero::reduce::{Dropped, ReducedAgents}; + use k8s_intf::gateway_agent_crd::GatewayAgent; + + fn handled_here(agent: &GatewayAgent, peering: &str) -> bool { + let Some(name) = agent.metadata.name.as_deref() else { + return false; + }; + let Some(group) = agent + .spec + .peerings + .as_ref() + .and_then(|peerings| peerings.get(peering)) + .and_then(|peering| peering.gateway_group.as_deref()) + else { + return false; + }; + agent + .spec + .groups + .as_ref() + .and_then(|groups| groups.get(group)) + .and_then(|group| group.members.as_ref()) + .is_some_and(|members| members.iter().any(|m| m.name == name)) + } + + fn difference(left: &[String], right: &[String]) -> Option { + if left == right { + return None; + } + let mut out: Vec = Vec::new(); + for line in left { + if !right.contains(line) { + out.push(format!(" only with it: {line}")); + } + } + for line in right { + if !left.contains(line) { + out.push(format!(" only without it: {line}")); + } + } + out.truncate(10); + Some(out.join("\n")) + } + + #[test] + fn every_expose_leaves_a_trace() { + static CHECKED: AtomicUsize = AtomicUsize::new(0); + static NOTHING_TO_DROP: AtomicUsize = AtomicUsize::new(0); + static REFUSED_WITHOUT: AtomicUsize = AtomicUsize::new(0); + static NOT_OURS: AtomicUsize = AtomicUsize::new(0); + static TRANSLATING: AtomicUsize = AtomicUsize::new(0); + + bolero::check!() + .with_generator(ReducedAgents::new( + k8s_intf::bolero::mutate::MutatedAgents::new( + k8s_intf::bolero::crd::GatewayAgentBuilder::new() + .families(vec![k8s_intf::bolero::AddressFamily::V4]) + .sizes(4, 3, 4, 3) + .build(), + ), + )) + .cloned() + .for_each( + |(mutation, agent, reduced, dropped): ( + Mutation, + GatewayAgent, + GatewayAgent, + Option, + )| { + let Some(dropped) = dropped else { + NOTHING_TO_DROP.fetch_add(1, Ordering::Relaxed); + return; + }; + let Ok(whole) = validator(&agent) else { + return; + }; + let Ok(less) = validator(&reduced) else { + REFUSED_WITHOUT.fetch_add(1, Ordering::Relaxed); + return; + }; + let (Some(with), Some(without)) = (Artifacts::of(&whole), Artifacts::of(&less)) + else { + return; + }; + + if !handled_here(&agent, &dropped.peering) { + NOT_OURS.fetch_add(1, Ordering::Relaxed); + return; + } + + CHECKED.fetch_add(1, Ordering::Relaxed); + + assert!( + difference(&with.frr, &without.frr).is_some(), + "{mutation:?}: removing {dropped} changed nothing in the routing \ + configuration, so the dataplane was never routing it" + ); + + let (table, name) = match dropped.nat { + None => return, + Some("static") => (&with.static_nat, &without.static_nat), + Some("port forwarding") => { + (&with.port_forwarding, &without.port_forwarding) + } + Some("masquerade") => (&with.masquerade, &without.masquerade), + Some(other) => unreachable!("unknown nat mode {other}"), + }; + TRANSLATING.fetch_add(1, Ordering::Relaxed); + assert!( + difference(table, name).is_some(), + "{mutation:?}: removing {dropped} changed nothing in the table for its own \ + NAT mode, so that translation was never installed" + ); + }, + ); + + let checked = CHECKED.load(Ordering::Relaxed); + let nothing = NOTHING_TO_DROP.load(Ordering::Relaxed); + let refused = REFUSED_WITHOUT.load(Ordering::Relaxed); + let not_ours = NOT_OURS.load(Ordering::Relaxed); + let translating = TRANSLATING.load(Ordering::Relaxed); + println!( + "{checked} exposes checked ({translating} of them translating); {nothing} \ + configurations had nothing to drop, {refused} became illegal without it, {not_ours} \ + sat in a peering this gateway does not handle" + ); + + let seen = checked + nothing + refused + not_ours; + #[cfg(not(fuzzing))] + if seen > 200 { + assert!( + checked * 40 > seen, + "only {checked} of {seen} cases got as far as comparing artifacts: this property has \ + become mostly skips" + ); + } + } +} diff --git a/nat/Cargo.toml b/nat/Cargo.toml index e46d2740a2..b2e69b14e7 100644 --- a/nat/Cargo.toml +++ b/nat/Cargo.toml @@ -36,7 +36,7 @@ shuttle = { workspace = true, optional = true } [dev-dependencies] # internal -config = { workspace = true } +config = { workspace = true, features = ["bolero"] } fixin = { workspace = true } test-utils = { workspace = true } lpm = { workspace = true, features = ["testing"] } diff --git a/nat/src/portfw/portfwtable/setup.rs b/nat/src/portfw/portfwtable/setup.rs index 812e14b329..0f1c27e34c 100644 --- a/nat/src/portfw/portfwtable/setup.rs +++ b/nat/src/portfw/portfwtable/setup.rs @@ -111,3 +111,56 @@ pub fn build_port_forwarding_configuration( } Ok(ruleset) } + +#[cfg(test)] +mod tests { + use super::*; + use config::external::overlay::vpcpeering::VpcExpose; + use config::external::overlay::vpcpeering::contract::{ + LOCAL_VNI, PortForwardingExpose, REMOTE_VNI, overlay_offering, + }; + + #[test] + fn an_expose_becomes_the_rules_it_describes() { + bolero::check!() + .with_generator(PortForwardingExpose) + .cloned() + .for_each(|expose: VpcExpose| { + let nat = expose.nat.as_ref().expect("port forwarding sets nat"); + let proto = nat.proto; + let internal = *expose.ips.first().expect("one prefix"); + let external = *nat.as_range.first().expect("one prefix"); + + let overlay = overlay_offering(expose.clone()).expect("overlay"); + let rules = build_port_forwarding_configuration(overlay.vpc_table()) + .expect("a validated port-forwarding expose should build"); + + let expected = if proto == L4Protocol::Any { 2 } else { 1 }; + assert_eq!(rules.len(), expected, "for {expose}"); + + for rule in &rules { + assert_eq!(rule.ext_prefix, external.prefix(), "external prefix"); + assert_eq!(rule.int_prefix, internal.prefix(), "internal prefix"); + assert_eq!( + rule.ext_ports.first().get(), + external.ports().expect("ports").start(), + "external ports" + ); + assert_eq!( + rule.int_ports.first().get(), + internal.ports().expect("ports").start(), + "internal ports" + ); + assert_eq!(rule.dst_vpcd, VpcDiscriminant::from_vni(vni(LOCAL_VNI))); + assert_eq!( + rule.key.src_vpcd(), + VpcDiscriminant::from_vni(vni(REMOTE_VNI)) + ); + } + }); + } + + fn vni(raw: u32) -> net::vxlan::Vni { + net::vxlan::Vni::new_checked(raw).unwrap_or_else(|_| unreachable!()) + } +} diff --git a/nat/src/static_nat/setup/mod.rs b/nat/src/static_nat/setup/mod.rs index 484f3f8d4e..df0e0c8e16 100644 --- a/nat/src/static_nat/setup/mod.rs +++ b/nat/src/static_nat/setup/mod.rs @@ -206,3 +206,94 @@ mod tests { .expect("Failed to build NAT tables"); } } + +#[cfg(test)] +mod config_driven { + use super::*; + use config::external::overlay::vpcpeering::VpcExpose; + use config::external::overlay::vpcpeering::contract::{ + LOCAL_VNI, REMOTE_VNI, StaticNatExpose, overlay_offering, + }; + use lpm::prefix::PrefixWithOptionalPorts; + use std::collections::BTreeSet; + use std::net::IpAddr; + + fn vni(raw: u32) -> Vni { + Vni::new_checked(raw).unwrap_or_else(|_| unreachable!()) + } + + fn addresses(prefixes: &BTreeSet) -> Vec { + let mut out = Vec::new(); + for prefix in prefixes { + let prefix = prefix.prefix(); + let (start, end) = (prefix.as_address(), prefix.last_address()); + let (mut bits, last) = match (start, end) { + (IpAddr::V4(a), IpAddr::V4(b)) => { + (u128::from(a.to_bits()), u128::from(b.to_bits())) + } + (IpAddr::V6(a), IpAddr::V6(b)) => (a.to_bits(), b.to_bits()), + _ => unreachable!("a prefix does not change address family"), + }; + while bits <= last { + out.push(match start { + IpAddr::V4(_) => IpAddr::V4( + u32::try_from(bits) + .unwrap_or_else(|_| unreachable!()) + .into(), + ), + IpAddr::V6(_) => IpAddr::V6(bits.into()), + }); + bits += 1; + } + } + out + } + + #[test] + fn the_two_sides_of_an_expose_map_one_to_one() { + bolero::check!() + .with_generator(StaticNatExpose) + .cloned() + .for_each(|expose: VpcExpose| { + let private = addresses(&expose.ips); + let public: BTreeSet = addresses( + &expose + .nat + .as_ref() + .expect("static nat sets nat") + .as_range + .clone(), + ) + .into_iter() + .collect(); + + let overlay = overlay_offering(expose.clone()).expect("overlay"); + let tables = build_nat_configuration(overlay.vpc_table()) + .expect("a validated expose builds"); + let table = tables + .get_table(vni(LOCAL_VNI)) + .expect("the offering vpc has a table"); + + let mut seen = BTreeSet::new(); + for source in &private { + let (mapped, _) = table + .find_src_mapping(source, None, vni(REMOTE_VNI)) + .unwrap_or_else(|| panic!("{source} has no mapping in {expose}")); + let mapped = mapped.inner(); + assert!( + public.contains(&mapped), + "{source} mapped to {mapped}, which the expose does not offer" + ); + assert!( + seen.insert(mapped), + "{source} mapped to {mapped}, which another address already took" + ); + } + assert_eq!( + seen.len(), + public.len(), + "the mapping left part of the public side unused, for {expose}" + ); + }); + } +}