RIB stats & AS Set flatten scripts (#115)

* use `--` to separate collector name

* empty rib_stats binary package

* up to getting rib file paths

* basic script to generate AS stats for each RIB

* naive stats generation

* single-pass stats generation

* handle partial failure

* correct hardcoded file path

* correct RIB file name filtering and collector name resolution

* skip processing if targets are present

* comment out empty oix route view

* script to flatten AS Sets

* fix AS Set flatten script

* name flatten_as_sets

download_ribs.py

@@ -79,13 +79,10 @@ def parallel_download(urls_filenames: list[tuple[str, str]]):
     # "route-views.siex": "https://archive.routeviews.org/route-views.siex/bgpdata", # Old
     # "route-views.ipv6": "https://archive.routeviews.org/ipv6", # Old
     # "route-views3-damp": "https://archive.routeviews.org/route-views3-damp", # Old
+    # "oix-route-views": "https://archive.routeviews.org/oix-route-views", # Empty
     # "oix-route-views-damp": "https://archive.routeviews.org/oix-route-views-damp", # Old
 }
 
-oix_route_view_collector2path = {
-    "oix-route-views": "https://archive.routeviews.org/oix-route-views",
-}
-
 ris_collectors = [
     "rrc00",
     "rrc01",
@@ -119,31 +116,21 @@ def route_view_download_tasks():
     return [
         (
             f"{url_path}/{YYYY}.{mm:02d}/RIBS/rib.{YYYY}{mm:02d}{dd:02d}.{HH:02d}00.bz2",
-            f"{DIR}/{collector}-rib.{YYYY}{mm:02d}{dd:02d}.{HH:02d}00.bz2",
+            f"{DIR}/{collector}--rib.{YYYY}{mm:02d}{dd:02d}.{HH:02d}00.bz2",
         )
         for collector, url_path in route_view_collector2path.items()
         for YYYY in years
         for mm in months
         for dd in days
         for HH in hours
-    ] + [
-        (
-            f"{url_path}/{YYYY}.{mm:02d}/oix-full-snapshot-{YYYY}-{mm:02d}-{dd:02d}-{HH:02d}00.bz2",
-            f"{DIR}/{collector}-oix-full-snapshot-{YYYY}-{mm:02d}-{dd:02d}-{HH:02d}00.bz2",
-        )
-        for collector, url_path in oix_route_view_collector2path.items()
-        for YYYY in years
-        for mm in months
-        for dd in days
-        for HH in hours
     ]
 
 
 def ripe_ris_download_tasks():
     return [
         (
             f"https://data.ris.ripe.net/{collector}/{YYYY}.{mm:02d}/bview.{YYYY}{mm:02d}{dd:02d}.{HH:02d}00.gz",
-            f"{DIR}/{collector}-bview.{YYYY}{mm:02d}{dd:02d}.{HH:02d}00.gz",
+            f"{DIR}/{collector}--bview.{YYYY}{mm:02d}{dd:02d}.{HH:02d}00.gz",
         )
         for collector in ris_collectors
         for YYYY in years

route_verification/Cargo.toml

@@ -10,6 +10,8 @@ members = [
     "lex",
     "parse",
     "shared_struct",
+    # Binary
+    "rib_stats",
 ]
 
 [workspace.dependencies]
@@ -49,6 +51,8 @@ lex = { package = "route_verification_lex", path = "./lex", version = "0.1.0" }
 parse = { package = "route_verification_parse", path = "./parse", version = "0.1.0" }
 shared_struct = { package = "route_verification_shared_struct", path = "./shared_struct", version = "0.1.0" }
 
+route_verification = { package = "route_verification", path = ".", version = "0.2.0" }
+
 [workspace.package]
 description = "Parse RPSL in the IRR to verify observed BGP routes"
 license = "MIT"
@@ -85,6 +89,7 @@ parse.workspace = true
 
 [dev-dependencies]
 dashmap.workspace = true
+hashbrown.workspace = true
 itertools.workspace = true
 net-literals.workspace = true
 polars.workspace = true

route_verification/bgp/src/stats.rs

@@ -8,8 +8,8 @@ use super::*;
 
 use Report::*;
 
-mod as_;
-mod as_pair;
+pub mod as_;
+pub mod as_pair;
 pub mod route;
 mod up_down_hill;
 
@@ -21,7 +21,7 @@ impl Compare {
     pub fn as_stats(&mut self, query: &QueryIr, db: &AsRelDb, map: &DashMap<u32, RouteStats<u64>>) {
         self.verbosity = Verbosity::all_stats();
         let reports = self.check_with_relationship(query, db);
-        for report in reports {
+        for report in &reports {
             as_::one(map, report);
         }
     }
@@ -44,7 +44,7 @@ impl Compare {
     ) {
         self.verbosity = Verbosity::all_stats();
         let reports = self.check_with_relationship(query, db);
-        for report in reports {
+        for report in &reports {
             as_pair::one(db, map, report);
         }
     }
@@ -53,7 +53,7 @@ impl Compare {
         self.verbosity = Verbosity::all_stats();
         let reports = self.check_with_relationship(query, db);
         let mut stats = RouteStats::default();
-        for report in reports {
+        for report in &reports {
             route::one(&mut stats, report);
         }
         stats

route_verification/bgp/src/stats/as_.rs

@@ -1,46 +1,46 @@
 use super::*;
 
-pub fn one(map: &DashMap<u32, RouteStats<u64>>, report: Report) {
+pub fn one(map: &DashMap<u32, RouteStats<u64>>, report: &Report) {
     match report {
-        OkImport { from: _, to } => map.entry(to).or_default().import_ok += 1,
-        OkExport { from, to: _ } => map.entry(from).or_default().export_ok += 1,
+        OkImport { from: _, to } => map.entry(*to).or_default().import_ok += 1,
+        OkExport { from, to: _ } => map.entry(*from).or_default().export_ok += 1,
         SkipImport { from: _, to, items } => {
-            let mut entry = map.entry(to).or_default();
+            let mut entry = map.entry(*to).or_default();
             entry.import_skip += 1;
             entry.skip(items);
         }
         SkipExport { from, to: _, items } => {
-            let mut entry = map.entry(from).or_default();
+            let mut entry = map.entry(*from).or_default();
             entry.export_skip += 1;
             entry.skip(items);
         }
         UnrecImport { from: _, to, items } => {
-            let mut entry = map.entry(to).or_default();
+            let mut entry = map.entry(*to).or_default();
             entry.import_unrec += 1;
             entry.unrec(items);
         }
         UnrecExport { from, to: _, items } => {
-            let mut entry = map.entry(from).or_default();
+            let mut entry = map.entry(*from).or_default();
             entry.export_unrec += 1;
             entry.unrec(items);
         }
         BadImport { from: _, to, items } => {
-            let mut entry = map.entry(to).or_default();
+            let mut entry = map.entry(*to).or_default();
             entry.import_err += 1;
             entry.bad(items);
         }
         BadExport { from, to: _, items } => {
-            let mut entry = map.entry(from).or_default();
+            let mut entry = map.entry(*from).or_default();
             entry.export_err += 1;
             entry.bad(items);
         }
         MehImport { from: _, to, items } => {
-            let mut entry = map.entry(to).or_default();
+            let mut entry = map.entry(*to).or_default();
             entry.import_meh += 1;
             entry.meh(items);
         }
         MehExport { from, to: _, items } => {
-            let mut entry = map.entry(from).or_default();
+            let mut entry = map.entry(*from).or_default();
             entry.export_meh += 1;
             entry.meh(items);
         }

route_verification/bgp/src/stats/as_pair.rs

@@ -1,51 +1,51 @@
 use super::*;
 
-pub fn one(db: &AsRelDb, map: &DashMap<(u32, u32), AsPairStats>, report: Report) {
+pub fn one(db: &AsRelDb, map: &DashMap<(u32, u32), AsPairStats>, report: &Report) {
     let entry = |from, to| {
         map.entry((from, to))
             .or_insert_with(|| AsPairStats::default_with_pair(from, to, db))
     };
 
     match report {
-        OkImport { from, to } => entry(from, to).route_stats.import_ok += 1,
-        OkExport { from, to } => entry(from, to).route_stats.export_ok += 1,
+        OkImport { from, to } => entry(*from, *to).route_stats.import_ok += 1,
+        OkExport { from, to } => entry(*from, *to).route_stats.export_ok += 1,
         SkipImport { from, to, items } => {
-            let mut entry = entry(from, to);
+            let mut entry = entry(*from, *to);
             entry.route_stats.import_skip += 1;
             entry.route_stats.skip(items)
         }
         SkipExport { from, to, items } => {
-            let mut entry = entry(from, to);
+            let mut entry = entry(*from, *to);
             entry.route_stats.export_skip += 1;
             entry.route_stats.skip(items)
         }
         UnrecImport { from, to, items } => {
-            let mut entry = entry(from, to);
+            let mut entry = entry(*from, *to);
             entry.route_stats.import_unrec += 1;
             entry.route_stats.unrec(items)
         }
         UnrecExport { from, to, items } => {
-            let mut entry = entry(from, to);
+            let mut entry = entry(*from, *to);
             entry.route_stats.export_unrec += 1;
             entry.route_stats.unrec(items)
         }
         BadImport { from, to, items } => {
-            let mut entry = entry(from, to);
+            let mut entry = entry(*from, *to);
             entry.route_stats.import_err += 1;
             entry.route_stats.bad(items)
         }
         BadExport { from, to, items } => {
-            let mut entry = entry(from, to);
+            let mut entry = entry(*from, *to);
             entry.route_stats.export_err += 1;
             entry.route_stats.bad(items)
         }
         MehImport { from, to, items } => {
-            let mut entry = entry(from, to);
+            let mut entry = entry(*from, *to);
             entry.route_stats.import_meh += 1;
             entry.route_stats.meh(items);
         }
         MehExport { from, to, items } => {
-            let mut entry = entry(from, to);
+            let mut entry = entry(*from, *to);
             entry.route_stats.export_meh += 1;
             entry.route_stats.meh(items);
         }

route_verification/bgp/src/stats/route.rs

@@ -1,7 +1,7 @@
 use super::*;
 use ReportItem::*;
 
-pub fn one(stats: &mut RouteStats<u16>, report: Report) {
+pub fn one(stats: &mut RouteStats<u16>, report: &Report) {
     match report {
         OkImport { from: _, to: _ } => stats.import_ok.inc(),
         OkExport { from: _, to: _ } => stats.export_ok.inc(),
@@ -145,7 +145,7 @@ pub struct RouteStats<T: Inc> {
 }
 
 impl<T: Inc> RouteStats<T> {
-    pub fn skip(&mut self, items: ReportItems) {
+    pub fn skip(&mut self, items: &ReportItems) {
         for item in items {
             match item {
                 SkipAsRegexWithTilde(_) => self.skip_regex_tilde.inc(),
@@ -156,11 +156,8 @@ impl<T: Inc> RouteStats<T> {
         }
     }
 
-    pub fn unrec(&mut self, items: ReportItems) {
-        if let Some(item) = items
-            .into_iter()
-            .reduce(|acc, e| if acc < e { acc } else { e })
-        {
+    pub fn unrec(&mut self, items: &ReportItems) {
+        if let Some(item) = items.iter().reduce(|acc, e| if acc < e { acc } else { e }) {
             match item {
                 UnrecImportEmpty => self.unrec_import_empty.inc(),
                 UnrecExportEmpty => self.unrec_export_empty.inc(),
@@ -177,11 +174,8 @@ impl<T: Inc> RouteStats<T> {
         }
     }
 
-    pub fn meh(&mut self, items: ReportItems) {
-        if let Some(item) = items
-            .into_iter()
-            .reduce(|acc, e| if acc < e { acc } else { e })
-        {
+    pub fn meh(&mut self, items: &ReportItems) {
+        if let Some(item) = items.iter().reduce(|acc, e| if acc < e { acc } else { e }) {
             match item {
                 SpecUphill => self.spec_uphill.inc(),
                 SpecUphillTier1 => self.spec_uphill_tier1.inc(),
@@ -199,7 +193,7 @@ impl<T: Inc> RouteStats<T> {
         }
     }
 
-    pub fn bad(&mut self, items: ReportItems) {
+    pub fn bad(&mut self, items: &ReportItems) {
         for item in items {
             match item {
                 MatchFilter => self.err_filter.inc(),

route_verification/rib_stats/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "rib_stats"
+version = "0.1.0"
+edition = "2021"
+description.workspace = true
+license.workspace = true
+repository.workspace = true
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow.workspace = true
+dashmap.workspace = true
+env_logger.workspace = true
+human-duration = "0.1"
+log.workspace = true
+rayon.workspace = true
+route_verification.workspace = true