Fetch configs in configuration package build script (#173)

* feature: build.rs fetches json from github pages

* feature: parallelize reqs in build.rs

* fix: handle failed http reqwests

* chore: changelog

* lint: remove unused import

* feature: config fetching on NomadConfig type

* refactor: make build.rs cleaner :)

* feature: re-run if configs are changed

* drive-by: clippy lints in tests

* feature: fetch from any URL

* fix: rename and relock

* drive-by: clippy lints in tests

agents/relayer/src/relayer.rs

@@ -292,7 +292,6 @@ mod test {
                 <Relayer as nomad_base::NomadAgent>::run(agent.build_channel("moonbeam"))
                     .await
                     .expect("Couldn't join relayer's run task");
-
             assert!(run_result.is_err(), "Must have returned error");
 
             let run_report_error_task = agent

configuration/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ### Unreleased
 
+- feature: pull config files from github pages instead of local versions
 - add environment variable overrides for agent configuration
 - add tests for agent environment variable overrides
 - remove `enabled` flag from agents project-wide

configuration/Cargo.toml

@@ -18,6 +18,8 @@ crate-type = ["cdylib", "rlib"]
 
 [build-dependencies]
 eyre = "0.6.6"
+reqwest = "0.11.10"
+tokio = { version = "1.18.2", features = ["rt", "macros"] }
 
 [dependencies]
 affix = "0.1.2"
@@ -29,6 +31,10 @@ serde_json = "1.0.78"
 serde_yaml = "0.8.23"
 nomad-types = { path = "../nomad-types" }
 
+[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
+reqwest = "0.11.10"
+
+
 [target.'cfg(target_arch = "wasm32")'.dependencies]
 wee_alloc = "0.4.5"
 js-sys = "0.3.56"

configuration/build.rs

@@ -1,21 +1,66 @@
-use std::io::Write;
+use std::{fs, io::Write, path::PathBuf};
 
 const DEFINITIONS: &str = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/data/definitions.ts"));
 const TYPEDEFS: &str = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/data/types.rs"));
 
 const OUTPUT_FILE: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/src/wasm/types.rs");
 
-fn main() -> eyre::Result<()> {
-    println!(
-        "cargo:rerun-if-changed={}",
-        concat!(env!("CARGO_MANIFEST_DIR"), "/data/definitions.ts")
-    );
-    println!(
-        "cargo:rerun-if-changed={}",
-        concat!(env!("CARGO_MANIFEST_DIR"), "/data/types.rs")
+const CONFIG_BASE_URI: &str = "https://nomad-xyz.github.io/config";
+const ENVS: &[&str] = &["development", "staging", "production"];
+const CONFIG_BASE_DIR: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/configs");
+
+fn env_json(env: &str) -> String {
+    format!("{}.json", env)
+}
+
+fn config_uri(env: &str) -> String {
+    format!("{}/{}", CONFIG_BASE_URI, env_json(env))
+}
+
+fn config_path(env: &str) -> PathBuf {
+    PathBuf::from(CONFIG_BASE_DIR).join(env_json(env))
+}
+
+async fn fetch_config(env: &str) -> eyre::Result<String> {
+    let uri = config_uri(env);
+    Ok(reqwest::get(uri).await?.text().await?)
+}
+
+fn store_config(env: &str, contents: &str) -> eyre::Result<()> {
+    let mut f = fs::OpenOptions::new()
+        .create(true)
+        .write(true)
+        .truncate(true)
+        .open(config_path(env))
+        .unwrap();
+
+    f.write_all(contents.as_ref())?;
+    Ok(())
+}
+
+async fn get_configs() -> eyre::Result<()> {
+    let (first, second, third) = tokio::join!(
+        fetch_config(ENVS[0]),
+        fetch_config(ENVS[1]),
+        fetch_config(ENVS[2]),
     );
 
-    let mut f = std::fs::OpenOptions::new()
+    // We do it this way so that if fetches fail we update all possible
+    // However, if disk access is erroring, we error
+    if let Ok(first) = first {
+        store_config(ENVS[0], &first)?
+    }
+    if let Ok(second) = second {
+        store_config(ENVS[1], &second)?
+    }
+    if let Ok(third) = third {
+        store_config(ENVS[2], &third)?;
+    }
+    Ok(())
+}
+
+fn gen_wasm_bindgen() -> eyre::Result<()> {
+    let mut f = fs::OpenOptions::new()
         .write(true)
         .truncate(true)
         .open(OUTPUT_FILE)
@@ -40,3 +85,32 @@ const _: &'static str = r#""###
 
     Ok(())
 }
+
+#[tokio::main]
+async fn main() -> eyre::Result<()> {
+    println!(
+        "cargo:rerun-if-changed={}",
+        concat!(env!("CARGO_MANIFEST_DIR"), "/data/definitions.ts")
+    );
+    println!(
+        "cargo:rerun-if-changed={}",
+        concat!(env!("CARGO_MANIFEST_DIR"), "/data/types.rs")
+    );
+    println!(
+        "cargo:rerun-if-changed={}",
+        concat!(env!("CARGO_MANIFEST_DIR"), "/configs/production.json")
+    );
+    println!(
+        "cargo:rerun-if-changed={}",
+        concat!(env!("CARGO_MANIFEST_DIR"), "/configs/development.json")
+    );
+    println!(
+        "cargo:rerun-if-changed={}",
+        concat!(env!("CARGO_MANIFEST_DIR"), "/configs/staging.json")
+    );
+
+    gen_wasm_bindgen()?;
+    get_configs().await?;
+
+    Ok(())
+}

configuration/configs/.gitignore

@@ -0,0 +1,2 @@
+*.json
+!test.json