flatten AS Sets into QueryAsSet when constructing Query

route_verification/bgp/src/cmp/as_regex.rs

@@ -49,10 +49,8 @@ impl<'a> AsRegex<'a> {
     /// Unrecorded ASNs are assigned `¿` to avoid being matched.
     pub fn asn_chars(&mut self, asn: u32, filter: Option<&Filter>, depth: isize) -> Vec<char> {
         let mut result: Vec<_> = self.interpreter.get_asn(asn).into_iter().collect();
-        let limit = self.c.cmp.recursion_limit;
-        let mut visited = visited();
         for (set, c) in self.interpreter.as_sets_with_char() {
-            match self.c.set_has_member(set, asn, limit, &mut visited) {
+            match self.c.set_has_member(set, asn) {
                 Ok(true) => result.push(c),
                 Ok(false) => (),
                 Err(r) => self.report |= r.unwrap(),

route_verification/bgp/src/cmp/filter.rs

@@ -259,7 +259,7 @@ impl<'a> CheckFilter<'a> {
         let last = self.last_on_path()?;
         if let (true, Ok(true)) = (
             op.permits(&self.cmp.prefix),
-            self.set_has_member(as_set, last, self.cmp.recursion_limit, &mut visited()),
+            self.set_has_member(as_set, last),
         ) {
             self.special_any_report(|| SpecAsSetContainsOriginButNoRoute(as_set.into(), last))
         } else {
@@ -331,40 +331,18 @@ impl<'a> CheckFilter<'a> {
         self.bad_any_report(|| RpslInvalidFilter(reason.into()))
     }
 
-    /// `Err` contains all the skips.
-    pub fn set_has_member(
-        &self,
-        set: &'a str,
-        asn: u32,
-        depth: isize,
-        visited: &mut BloomHashSet<&'a str>,
-    ) -> Result<bool, AnyReport> {
-        if depth < 0 {
-            return Err(bad_any_report(RecCheckSetMember(set.into())));
-        }
-        let hash = visited.make_hash(&set);
-        if visited.contains_with_hash(&set, hash) {
-            return Err(empty_bad_any_report());
-        }
+    /// `Err` contains all the skips in an [`AnyReport`].
+    pub fn set_has_member(&self, set: &'a str, asn: u32) -> Result<bool, AnyReport> {
         let as_set = match self.query.as_sets.get(set) {
             Some(s) => s,
             None => return Err(self.unrec_any_report(|| UnrecordedAsSet(set.into()))),
         };
-        if as_set.is_any || as_set.members.contains(&asn) {
-            return Ok(true);
-        }
-        let mut report = SkipAnyReport(vec![]);
-        visited.insert_with_hash(set, hash);
-        for set in &as_set.set_members {
-            match self.set_has_member(set, asn, depth - 1, visited) {
-                Ok(true) => return Ok(true),
-                Ok(false) => (),
-                Err(err) => report |= err.unwrap(),
-            }
-        }
-        match report {
-            SkipAnyReport(items) if items.is_empty() => Ok(false),
-            report => Err(Some(report)),
+        if as_set.contains(&asn) {
+            Ok(true)
+        } else if !as_set.unrecorded_members.is_empty() {
+            Err(self.unrec_any_report(|| UnrecordedSomeAsSet(set.into())))
+        } else {
+            Ok(false)
         }
     }
 }

route_verification/bgp/src/cmp/peering.rs

@@ -68,9 +68,7 @@ impl<'a> CheckPeering<'a> {
         match as_name {
             AsName::Any => None,
             AsName::Num(num) => self.check_remote_as_num(*num),
-            AsName::Set(name) => {
-                self.check_remote_as_set(name, depth, &mut BloomHashSet::with_capacity(2048, 32768))
-            }
+            AsName::Set(name) => self.check_remote_as_set(name),
             AsName::Invalid(reason) => self.bad_any_report(|| RpslInvalidAsName(reason.into())),
         }
     }
@@ -83,52 +81,23 @@ impl<'a> CheckPeering<'a> {
         }
     }
 
-    fn check_remote_as_set(
-        &self,
-        name: &'a str,
-        depth: isize,
-        visited: &mut BloomHashSet<&'a str>,
-    ) -> AnyReport {
-        let hash = visited.make_hash(&name);
-        if visited.contains_with_hash(&name, hash) {
-            return empty_bad_any_report();
-        }
-
-        if depth <= 0 {
-            return bad_any_report(RecRemoteAsSet(name.into()));
-        }
+    fn check_remote_as_set(&self, name: &'a str) -> AnyReport {
         let as_set = match self.c.query.as_sets.get(name) {
             Some(r) => r,
             None => return self.unrec_any_report(|| UnrecordedAsSet(name.into())),
         };
 
-        if as_set.is_any || as_set.members.binary_search(&self.accept_num).is_ok() {
+        if as_set.contains(&self.accept_num) {
             return None;
         }
 
-        self.check_remote_as_set_members(name, depth, visited, hash, as_set)
-    }
-
-    fn check_remote_as_set_members(
-        &self,
-        name: &'a str,
-        depth: isize,
-        visited: &mut BloomHashSet<&'a str>,
-        hash: u64,
-        as_set: &'a AsSet,
-    ) -> AnyReport {
-        visited.insert_with_hash(name, hash);
-
-        let mut report = AnyReportCase::const_default();
-        for set in &as_set.set_members {
-            report |= self.check_remote_as_set(set, depth - 1, visited)?;
-        }
-        if let BadAnyReport(_) = report {
+        if as_set.unrecorded_members.is_empty() {
             self.bad_any_report(|| MatchRemoteAsSet(name.into()))
         } else {
-            Some(report)
+            self.unrec_any_report(|| UnrecordedAsSet(name.into()))
         }
     }
+
     fn check_remote_peering_set(&self, name: &str, depth: isize) -> AnyReport {
         if depth <= 0 {
             return bad_any_report(RecRemotePeeringSet(name.into()));

route_verification/bgp/src/lib.rs

@@ -20,7 +20,7 @@ pub mod wrapper;
 pub use {
     bgpmap::{self as map, AsPathEntry},
     cmp::Compare,
-    query::{customer_set, AsProperty, AsSetRoute, QueryIr},
+    query::{customer_set, AsProperty, AsSetRoute, QueryAsSet, QueryIr},
     report::{Report, ReportItem},
     verbosity::Verbosity,
     wrapper::{parse_mrt, Line},

route_verification/bgp/src/query.rs

@@ -1,4 +1,4 @@
-use hashbrown::HashMap;
+use hashbrown::{HashMap, HashSet};
 
 use super::*;
 
@@ -48,6 +48,25 @@ impl AsSetRoute {
     }
 }
 
+#[derive(Clone, Debug, Default, Eq, PartialEq)]
+pub struct QueryAsSet {
+    pub body: String,
+    pub members: HashSet<u32>,
+    pub unrecorded_members: Vec<String>,
+    pub is_any: bool,
+}
+
+impl QueryAsSet {
+    pub fn contains(&self, as_num: &u32) -> bool {
+        self.is_any || self.members.contains(as_num)
+    }
+
+    pub fn clean_up(&mut self) {
+        self.members.shrink_to_fit();
+        clean_vec(&mut self.unrecorded_members);
+    }
+}
+
 pub fn clean_vec<T: Ord>(v: &mut Vec<T>) {
     v.sort();
     v.shrink_to_fit();
@@ -93,7 +112,7 @@ fn all_providers(versions: &Versions, num: u32, db: &AsRelDb) -> bool {
 #[derive(Clone, Debug, Default, Eq, PartialEq)]
 pub struct QueryIr {
     pub aut_nums: HashMap<u32, AutNum>,
-    pub as_sets: HashMap<String, AsSet>,
+    pub as_sets: HashMap<String, QueryAsSet>,
     pub route_sets: HashMap<String, RouteSet>,
     pub peering_sets: HashMap<String, PeeringSet>,
     pub filter_sets: HashMap<String, FilterSet>,
@@ -128,7 +147,7 @@ impl QueryIr {
         let as_set_routes = flatten_as_set_routes(&as_set_routes);
         let as_set_routes = HashMap::from_iter(as_set_routes);
         let aut_nums = HashMap::from_iter(aut_nums);
-        let as_sets = HashMap::from_iter(as_sets);
+        let as_sets = flatten_as_sets(&as_sets);
         let route_sets = HashMap::from_iter(route_sets);
         let peering_sets = HashMap::from_iter(peering_sets);
         let filter_sets = HashMap::from_iter(filter_sets);
@@ -184,3 +203,49 @@ fn flatten_as_set_routes(
     });
     result
 }
+
+fn flatten_as_set(
+    query_as_set: &mut QueryAsSet,
+    visited_sets: &mut HashSet<String>,
+    set_members: &[String],
+    as_sets: &BTreeMap<String, AsSet>,
+) {
+    for set_member in set_members {
+        if !visited_sets.contains(set_member) {
+            visited_sets.insert(set_member.to_string());
+            if let Some(set) = as_sets.get(set_member) {
+                query_as_set.members.extend(set.members.iter().copied());
+            } else {
+                query_as_set.unrecorded_members.push(set_member.to_string());
+            }
+        }
+    }
+}
+
+pub fn flatten_as_sets(as_sets: &BTreeMap<String, AsSet>) -> HashMap<String, QueryAsSet> {
+    as_sets
+        .par_iter()
+        .map(|(name, set)| {
+            let AsSet {
+                body,
+                members,
+                set_members,
+                is_any,
+            } = set;
+            let mut query_as_set = QueryAsSet {
+                body: body.clone(),
+                members: HashSet::with_capacity(set_members.len() * 32 + members.len()),
+                unrecorded_members: Vec::new(),
+                is_any: *is_any,
+            };
+            query_as_set.members.extend(members.iter().copied());
+
+            let mut visited = HashSet::with_capacity(set_members.len() * 8);
+            visited.insert(name.to_string());
+            flatten_as_set(&mut query_as_set, &mut visited, set_members, as_sets);
+
+            query_as_set.clean_up();
+            (name.to_owned(), query_as_set)
+        })
+        .collect()
+}

route_verification/bgp/src/report.rs

@@ -103,6 +103,7 @@ pub enum ReportItem {
 
     // Unrecorded RPSL Objects.
     UnrecordedAsSet(String),
+    UnrecordedSomeAsSet(String),
     UnrecordedAsRoutes(u32),
     UnrecordedAsSetRoute(String),
     UnrecordedSomeAsSetRoute(String),

route_verification/bgp/src/tests/query.rs

@@ -1,3 +1,4 @@
+use hashbrown::HashSet;
 use maplit::hashmap;
 
 use super::{cmp::*, *};
@@ -6,12 +7,18 @@ use super::{cmp::*, *};
 fn psedo_customer_set() -> Result<()> {
     let ir = ir()?;
     let db = as_relationship_db()?;
-    let mut actual = HashMap::from_iter(QueryIr::from_ir_and_as_relationship(ir, &db).as_sets);
-    actual.iter_mut().for_each(|(_, v)| v.members.sort());
+    let actual = HashMap::from_iter(QueryIr::from_ir_and_as_relationship(ir, &db).as_sets);
     assert_eq!(actual, expected_as_sets());
     Ok(())
 }
 
-fn expected_as_sets() -> HashMap<String, AsSet> {
-    hashmap! {"c#2914".into()=> AsSet { body: "".into(), members: vec![4096, 9583], set_members: vec![], is_any: false }, "c#1239".into()=> AsSet { body: "".into(), members: vec![3130], set_members: vec![], is_any: false }}
+fn expected_as_sets() -> HashMap<String, QueryAsSet> {
+    hashmap! {
+        "c#2914".into()=> QueryAsSet {
+            body: "".into(), members:  HashSet::from([4096, 9583]), unrecorded_members: vec![], is_any: false
+        },
+        "c#1239".into()=> QueryAsSet {
+            body: "".into(), members: HashSet::from([3130]), unrecorded_members: vec![], is_any: false
+        }
+    }
 }

route_verification/src/evcxr_examples.rs

@@ -20,7 +20,6 @@ mod count_asn_in_peering;
 mod count_path_sets;
 mod count_router_info;
 mod filter_as;
-mod flatten_as_set;
 mod route_stats;
 mod specific_line;
 

route_verification/src/evcxr_examples/as_set_graphing.rs

@@ -2,7 +2,7 @@ use super::*;
 
 /// Fully flatten each AS Set to all of its members.
 /// Copy from the `:dep` line after running code from [`parse_bgp_lines`].
-fn as_sets_graph_stats(query: QueryIr) -> Result<()> {
+fn as_sets_graph_stats(query: Ir) -> Result<()> {
     use graph as route_verification_graph;
     /*
     :dep route_verification_graph = { path = "route_verification/graph" }
@@ -12,7 +12,7 @@ fn as_sets_graph_stats(query: QueryIr) -> Result<()> {
         as_set_graph: &mut ASSetGraph,
         as_num_or_set: ASNumOrSet,
         as_set: &AsSet,
-        as_sets: &HashMap<String, AsSet>,
+        as_sets: &std::collections::BTreeMap<String, AsSet>,
     ) {
         let set_members_cloned: Vec<ASNumOrSet> = as_set
             .set_members