From 422489d5d118ee6eed69228cda0a42645a292732 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Tue, 29 Sep 2020 12:18:40 +0200
Subject: [PATCH 001/145] add k8s structure

---
 turbolift_internals/Cargo.toml        |  8 ++-
 turbolift_internals/src/kubernetes.rs | 75 ++++++++++++++++++++++++++-
 2 files changed, 81 insertions(+), 2 deletions(-)

diff --git a/turbolift_internals/Cargo.toml b/turbolift_internals/Cargo.toml
index 974ef845..19bddf1b 100644
--- a/turbolift_internals/Cargo.toml
+++ b/turbolift_internals/Cargo.toml
@@ -28,4 +28,10 @@ lazy_static = "1"
 anyhow = "1"
 cached = "0.19"
 async-std = "1.6"
-async-trait = "0.1"
\ No newline at end of file
+async-trait = "0.1"
+
+
+# kubernetes-specific requirements
+kube = "0.42.0"
+kube-runtime = "0.42.0"
+k8s-openapi = { version = "0.9.0", default-features = false, features = ["v1_17"] }
\ No newline at end of file
diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 00bca357..a0fcd310 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -1 +1,74 @@
-pub struct K8s {}
+use async_trait::async_trait;
+use futures::{StreamExt, TryStreamExt};
+use kube::api::{Api, Meta, ListParams, PostParams, WatchEvent};
+use kube::Client;
+use k8s_openapi::api::core::v1::Pod;
+use cached::proc_macro::cached;
+use url::Url;
+
+use crate::distributed_platform::{
+    ArgsString, DistributionPlatform, DistributionResult, JsonResponse,
+};
+
+const K8S_NAMESPACE: &str = "turbolift";
+type ImageTag = String;
+
+pub type K8sConfig = kube::config::Config;
+
+pub struct K8s {
+    config: K8sConfig,
+    pods: Vec<Pod>,
+}
+
+#[async_trait]
+impl DistributionPlatform for K8s {
+    fn declare(&mut self, function_name: &str, project_tar: &[u8]) {
+        // connect to cluster. tries in-cluster configuration first, then falls back to kubeconfig file.
+        let client = Client::try_default().await?;
+        let pods: Api<Pod> = Api::namespaced(client, K8S_NAMESPACE);
+
+        // generate image & host it on a local repo
+        let repo_url = setup_repo().expect("error initializing network repository");
+        let local_tag = make_image(function_name, project_tar).expect("error making image");
+        let tag_in_repo = add_image_to_repo(local_tag).expect("error adding image to repo");
+        let image_url = repo_url.join(&tag_in_repo).expect("url parse error");
+
+        // make pod
+        let pod_name = function_name;
+        let container_name = function_name.to_string() + "-container";
+        let pod = serde_json::from_value(serde_json::json!({
+            "apiVersion": "v1",
+            "kind": "Pod",
+            "metadata": {
+                "name": pod_name
+            },
+            "spec": {
+                "containers": [
+                    {
+                        "name": container_name,
+                        "image": image_url.as_str(),
+                    },
+                ],
+            }
+        }))?;
+        self.pods.push(pods.create(&PostParams::default(), &pod).await?);
+        // todo do we need to monitor the pod in any way??
+    }
+
+    async fn dispatch(&mut self, function_name: &str, params: ArgsString) -> DistributionResult<JsonResponse> {
+        unimplemented!()
+    }
+}
+
+#[cached(size=1)]
+fn setup_repo() -> DistributionResult<Url> {
+    unimplemented!()
+}
+
+fn make_image(function_name: &str, project_tar: &[u8]) -> DistributionResult<ImageTag> {
+    unimplemented!()
+}
+
+fn add_image_to_repo(local_tag: ImageTag) -> DistributionResult<ImageTag> {
+    unimplemented!()
+}
\ No newline at end of file

From a7c208c5632479707d063cc5200d507e31665da6 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Tue, 29 Sep 2020 20:11:14 +0200
Subject: [PATCH 002/145] make declare async

---
 .../src/distributed_platform.rs               |  2 +-
 turbolift_internals/src/kubernetes.rs         | 40 ++++++++++---------
 turbolift_internals/src/local_queue.rs        |  9 +++--
 turbolift_macros/src/lib.rs                   | 13 ++++--
 4 files changed, 38 insertions(+), 26 deletions(-)

diff --git a/turbolift_internals/src/distributed_platform.rs b/turbolift_internals/src/distributed_platform.rs
index 22aab0ba..2a8a36c1 100644
--- a/turbolift_internals/src/distributed_platform.rs
+++ b/turbolift_internals/src/distributed_platform.rs
@@ -11,7 +11,7 @@ pub type JsonResponse = String;
 #[async_trait]
 pub trait DistributionPlatform {
     /// declare a function
-    fn declare(&mut self, function_name: &str, project_tar: &[u8]);
+    async fn declare(&mut self, function_name: &str, project_tar: &[u8]) -> DistributionResult<()>;
 
     // dispatch params to a function
     async fn dispatch(
diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index a0fcd310..c70052f3 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -1,9 +1,8 @@
 use async_trait::async_trait;
-use futures::{StreamExt, TryStreamExt};
-use kube::api::{Api, Meta, ListParams, PostParams, WatchEvent};
-use kube::Client;
-use k8s_openapi::api::core::v1::Pod;
 use cached::proc_macro::cached;
+use k8s_openapi::api::core::v1::Pod;
+use kube::api::{Api, PostParams};
+use kube::Client;
 use url::Url;
 
 use crate::distributed_platform::{
@@ -13,25 +12,24 @@ use crate::distributed_platform::{
 const K8S_NAMESPACE: &str = "turbolift";
 type ImageTag = String;
 
-pub type K8sConfig = kube::config::Config;
+// pub type K8sConfig = kube::config::Config; todo
 
 pub struct K8s {
-    config: K8sConfig,
     pods: Vec<Pod>,
 }
 
 #[async_trait]
 impl DistributionPlatform for K8s {
-    fn declare(&mut self, function_name: &str, project_tar: &[u8]) {
+    async fn declare(&mut self, function_name: &str, project_tar: &[u8]) -> DistributionResult<()> {
         // connect to cluster. tries in-cluster configuration first, then falls back to kubeconfig file.
         let client = Client::try_default().await?;
         let pods: Api<Pod> = Api::namespaced(client, K8S_NAMESPACE);
 
         // generate image & host it on a local repo
-        let repo_url = setup_repo().expect("error initializing network repository");
-        let local_tag = make_image(function_name, project_tar).expect("error making image");
-        let tag_in_repo = add_image_to_repo(local_tag).expect("error adding image to repo");
-        let image_url = repo_url.join(&tag_in_repo).expect("url parse error");
+        let repo_url = setup_repo();
+        let local_tag = make_image(function_name, project_tar)?;
+        let tag_in_repo = add_image_to_repo(local_tag)?;
+        let image_url = repo_url.join(&tag_in_repo)?;
 
         // make pod
         let pod_name = function_name;
@@ -51,24 +49,30 @@ impl DistributionPlatform for K8s {
                 ],
             }
         }))?;
-        self.pods.push(pods.create(&PostParams::default(), &pod).await?);
+        self.pods
+            .push(pods.create(&PostParams::default(), &pod).await?);
         // todo do we need to monitor the pod in any way??
+        Ok(())
     }
 
-    async fn dispatch(&mut self, function_name: &str, params: ArgsString) -> DistributionResult<JsonResponse> {
+    async fn dispatch(
+        &mut self,
+        _function_name: &str,
+        _params: ArgsString,
+    ) -> DistributionResult<JsonResponse> {
         unimplemented!()
     }
 }
 
-#[cached(size=1)]
-fn setup_repo() -> DistributionResult<Url> {
+#[cached(size = 1)]
+fn setup_repo() -> Url {
     unimplemented!()
 }
 
-fn make_image(function_name: &str, project_tar: &[u8]) -> DistributionResult<ImageTag> {
+fn make_image(_function_name: &str, _project_tar: &[u8]) -> DistributionResult<ImageTag> {
     unimplemented!()
 }
 
-fn add_image_to_repo(local_tag: ImageTag) -> DistributionResult<ImageTag> {
+fn add_image_to_repo(_local_tag: ImageTag) -> DistributionResult<ImageTag> {
     unimplemented!()
-}
\ No newline at end of file
+}
diff --git a/turbolift_internals/src/local_queue.rs b/turbolift_internals/src/local_queue.rs
index a3ac54e3..712293c6 100644
--- a/turbolift_internals/src/local_queue.rs
+++ b/turbolift_internals/src/local_queue.rs
@@ -35,19 +35,20 @@ impl LocalQueue {
 #[async_trait]
 impl DistributionPlatform for LocalQueue {
     /// declare a function. Runs once.
-    fn declare(&mut self, function_name: &str, project_tar: &[u8]) {
+    async fn declare(&mut self, function_name: &str, project_tar: &[u8]) -> DistributionResult<()> {
         let relative_build_dir = Path::new(".")
             .join(".turbolift")
             .join(".worker_build_cache");
-        fs::create_dir_all(&relative_build_dir).unwrap();
-        let build_dir = relative_build_dir.canonicalize().unwrap();
+        fs::create_dir_all(&relative_build_dir)?;
+        let build_dir = relative_build_dir.canonicalize()?;
         decompress_proj_src(project_tar, &build_dir).unwrap();
         let function_executable =
             Path::new(CACHE_PATH.as_os_str()).join(function_name.to_string() + "_server");
-        make_executable(&build_dir.join(function_name), Some(&function_executable)).unwrap();
+        make_executable(&build_dir.join(function_name), Some(&function_executable))?;
         self.fn_name_to_binary_path
             .insert(function_name.to_string(), function_executable);
         //std::fs::remove_dir_all(build_dir.join(function_name)).unwrap(); todo
+        Ok(())
     }
 
     // dispatch params to a function. Runs each time the function is called.
diff --git a/turbolift_macros/src/lib.rs b/turbolift_macros/src/lib.rs
index 1e9c8ad1..2922e628 100644
--- a/turbolift_macros/src/lib.rs
+++ b/turbolift_macros/src/lib.rs
@@ -40,6 +40,10 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
         &function_name_string,
         untyped_params,
     );
+    let declaration_error_text = format!(
+        "turbolift: error while declaring {}",
+        original_target_function_name
+    );
 
     // todo extract any docs from passed function and put into fn wrapper
 
@@ -154,18 +158,21 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
             turbolift::DistributionResult<#result_type> {
             use std::time::Duration;
             use turbolift::DistributionPlatform;
+            use turbolift::DistributionResult;
             use turbolift::async_std::task;
             use turbolift::cached::proc_macro::cached;
 
             // call .declare once by memoizing the call
             #[cached(size=1)]
-            fn setup() {
+            async fn setup() -> DistributionResult<()> {
                 #distribution_platform
                     .lock()
                     .unwrap()
-                    .declare(#original_target_function_name, #project_source_binary);
+                    .declare(#original_target_function_name, #project_source_binary)
+                    .await
+                    .expect(#declaration_error_text)
             }
-            setup();
+            setup().await?;
 
             let params = #params_vec.join("/");
 

From fcd2f511a799c43a86ae3171c0acf10c170a1bd1 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Tue, 29 Sep 2020 22:57:32 +0200
Subject: [PATCH 003/145] add docker test framework for k8s

---
 .github/workflows/docker.yml                   | 10 +++++++++-
 .gitignore                                     |  3 ++-
 examples/kubernetes_example/Cargo.toml         | 11 +++++++++++
 examples/kubernetes_example/Dockerfile         | 13 +++++++++++++
 examples/kubernetes_example/example_runtime.sh | 15 +++++++++++++++
 examples/kubernetes_example/src/main.rs        |  3 +++
 turbolift_internals/src/kubernetes.rs          |  2 --
 7 files changed, 53 insertions(+), 4 deletions(-)
 create mode 100644 examples/kubernetes_example/Cargo.toml
 create mode 100644 examples/kubernetes_example/Dockerfile
 create mode 100644 examples/kubernetes_example/example_runtime.sh
 create mode 100644 examples/kubernetes_example/src/main.rs

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 9773b997..48367826 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -12,4 +12,12 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: build local queue example
-        run: docker build -f examples/local_queue_example/Dockerfile .
\ No newline at end of file
+        run: docker build -f examples/local_queue_example/Dockerfile .
+  kubernetes_example:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: build k8s example
+        run: docker build -f examples/kubernetes_example/Dockerfile -t kub .
+      - name: run k8s example
+        run: docker run -v /var/run/docker.sock:/var/run/docker.sock kub
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index fa78f90a..1af1252e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,5 @@
 /target
 target/
 Cargo.lock
-.turbolift/
\ No newline at end of file
+.turbolift/
+.idea
\ No newline at end of file
diff --git a/examples/kubernetes_example/Cargo.toml b/examples/kubernetes_example/Cargo.toml
new file mode 100644
index 00000000..5ee7c45a
--- /dev/null
+++ b/examples/kubernetes_example/Cargo.toml
@@ -0,0 +1,11 @@
+[package]
+name = "kubernetes_example"
+version = "0.1.0"
+authors = ["Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>"]
+edition = "2018"
+
+[features]
+"distributed" = ["turbolift/distributed"]
+
+[dependencies]
+turbolift = { path="../../" }
\ No newline at end of file
diff --git a/examples/kubernetes_example/Dockerfile b/examples/kubernetes_example/Dockerfile
new file mode 100644
index 00000000..6a05bdee
--- /dev/null
+++ b/examples/kubernetes_example/Dockerfile
@@ -0,0 +1,13 @@
+# docker build -f examples/kubernetes_example/Dockerfile -t kub .
+# docker run -v /var/run/docker.sock:/var/run/docker.sock kub
+
+# set up kind (k8s in docker)
+FROM rustlang/rust:nightly
+RUN  apt-get update \
+  && apt-get install -y wget \
+  && apt-get install -y docker.io \
+  && rm -rf /var/lib/apt/lists/*
+RUN wget -O ./kind --show-progress "https://kind.sigs.k8s.io/dl/v0.9.0/kind-$(uname)-amd64"
+RUN chmod +x ./kind
+COPY ./ ./
+CMD sh examples/kubernetes_example/example_runtime.sh
\ No newline at end of file
diff --git a/examples/kubernetes_example/example_runtime.sh b/examples/kubernetes_example/example_runtime.sh
new file mode 100644
index 00000000..cb89e8b6
--- /dev/null
+++ b/examples/kubernetes_example/example_runtime.sh
@@ -0,0 +1,15 @@
+# error if any command fails
+set -e
+
+# setup cluster (will be used in all tests & runs)
+./kind create cluster
+
+cd examples/kubernetes_example
+
+# test
+RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test
+RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test --features "distributed"
+
+# run
+RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run
+RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run --features "distributed"
\ No newline at end of file
diff --git a/examples/kubernetes_example/src/main.rs b/examples/kubernetes_example/src/main.rs
new file mode 100644
index 00000000..e7a11a96
--- /dev/null
+++ b/examples/kubernetes_example/src/main.rs
@@ -0,0 +1,3 @@
+fn main() {
+    println!("Hello, world!");
+}
diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index c70052f3..3dfb0b89 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -12,8 +12,6 @@ use crate::distributed_platform::{
 const K8S_NAMESPACE: &str = "turbolift";
 type ImageTag = String;
 
-// pub type K8sConfig = kube::config::Config; todo
-
 pub struct K8s {
     pods: Vec<Pod>,
 }

From 5d8cfc0c95ea801a923d6028a8b78746d610ccd8 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Wed, 30 Sep 2020 11:51:31 +0200
Subject: [PATCH 004/145] use docker suite to run example tests, use rust suite
 for linting

---
 .github/workflows/rust.yml              | 6 +-----
 examples/local_queue_example/Dockerfile | 8 +++++++-
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 4d9294c1..648ae6b3 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -10,7 +10,7 @@ env:
   CARGO_TERM_COLOR: always
 
 jobs:
-  test:
+  lints:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
@@ -30,7 +30,3 @@ jobs:
       run: RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo check --features "distributed"
     - name: Clippy
       run: RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo clippy -- -D warnings
-    - name: Local Queue without distributed feature
-      run: cd examples/local_queue_example && RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo test
-    - name: Local Queue with distributed feature
-      run: cd examples/local_queue_example && RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo test --features "distributed"
diff --git a/examples/local_queue_example/Dockerfile b/examples/local_queue_example/Dockerfile
index 56d976cb..be9a5c9b 100644
--- a/examples/local_queue_example/Dockerfile
+++ b/examples/local_queue_example/Dockerfile
@@ -2,5 +2,11 @@
 FROM rustlang/rust:nightly
 COPY ./ ./
 WORKDIR examples/local_queue_example
+
+# test
 RUN RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test
-RUN RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test --features "distributed"
\ No newline at end of file
+RUN RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test --features "distributed"
+
+# run
+RUN RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run
+RUN RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run --features "distributed"
\ No newline at end of file

From 5bc250625486533c6d5de498f393e69df2dc8e7e Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Wed, 30 Sep 2020 11:59:33 +0200
Subject: [PATCH 005/145] lints: make sure that subprojects are formatted
 correctly

---
 .github/workflows/rust.yml       | 6 +++++-
 turbolift_internals/src/lib.rs   | 4 ++--
 turbolift_internals/src/utils.rs | 2 --
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 648ae6b3..cc93f87c 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -22,8 +22,12 @@ jobs:
           override: true
     - name: Rustup
       run: rustup update
-    - name: Format
+    - name: Format turbolift
       run: RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo fmt
+    - name: Format turbolift_internals
+      run: cd turbolift_internals && RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo fmt
+    - name: Format turbolift_macros
+      run: cd turbolift_macros && RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo
     - name: Check without distributed feature
       run: RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo check
     - name: Check with distributed feature
diff --git a/turbolift_internals/src/lib.rs b/turbolift_internals/src/lib.rs
index 586cdb01..b8ccde96 100644
--- a/turbolift_internals/src/lib.rs
+++ b/turbolift_internals/src/lib.rs
@@ -5,9 +5,9 @@ use std::path::Path;
 pub mod build_project;
 pub mod distributed_platform;
 pub mod extract_function;
-pub mod utils;
-pub mod local_queue;
 pub mod kubernetes;
+pub mod local_queue;
+pub mod utils;
 pub use serde_json;
 
 lazy_static! {
diff --git a/turbolift_internals/src/utils.rs b/turbolift_internals/src/utils.rs
index 4abc4b86..52e10e44 100644
--- a/turbolift_internals/src/utils.rs
+++ b/turbolift_internals/src/utils.rs
@@ -3,5 +3,3 @@ pub use std::os::unix::fs::symlink as symlink_dir;
 
 #[cfg(target_family = "windows")]
 pub use std::os::windows::fs::symlink_dir;
-
-

From 454462047b69d6ca2292f9d2a4b2089554a92b34 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Wed, 30 Sep 2020 17:21:35 +0200
Subject: [PATCH 006/145] bug fix: make sure that distributed feature is
 actually being triggered also make sure that actix-web is available when
 needed

---
 Cargo.toml                  | 6 +++---
 src/lib.rs                  | 1 -
 turbolift_macros/Cargo.toml | 3 +++
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 85022eea..63091ce2 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -8,8 +8,8 @@ readme = "README.md"
 license = "Hippocratic-2.1"
 
 [features]
-distributed = ["chrono", "cached", "async-std"]
-service = ["actix-web", "serde_json"]
+distributed = ["chrono", "cached", "async-std", "turbolift_macros/distributed"]
+service = ["serde_json"]
 # todo we can optimize reqs for children with this load
 
 [dependencies]
@@ -18,5 +18,5 @@ turbolift_internals = { path = "./turbolift_internals" }
 async-std = { version = "1.6", optional = true }
 chrono = { version = "0.4", optional = true }
 cached = { version = "0.19", optional = true }
-actix-web = { version = "3", optional = true }
+actix-web = { version = "3" }
 serde_json = { version = "1", optional = true }
\ No newline at end of file
diff --git a/src/lib.rs b/src/lib.rs
index 3a0bd11f..1253f800 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -5,7 +5,6 @@ pub use cached;
 #[cfg(feature = "distributed")]
 pub use chrono;
 
-#[cfg(feature = "service")]
 pub use actix_web;
 #[cfg(feature = "service")]
 pub use serde_json;
diff --git a/turbolift_macros/Cargo.toml b/turbolift_macros/Cargo.toml
index aac937be..c693e1d6 100644
--- a/turbolift_macros/Cargo.toml
+++ b/turbolift_macros/Cargo.toml
@@ -9,6 +9,9 @@ edition = "2018"
 [lib]
 proc-macro = true
 
+[features]
+"distributed" = []
+
 [dependencies]
 quote = "1"
 proc-macro2 = "1"

From a770b71455cce46a40f2f824e32a725194cf6da1 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Wed, 30 Sep 2020 17:28:24 +0200
Subject: [PATCH 007/145] add k8s example structure the internals haven't been
 implemented yet, but I want an interface to do TDD off of while developing

---
 Cargo.toml                                    |  4 +-
 examples/kubernetes_example/Cargo.toml        |  5 ++
 .../kubernetes_example/example_runtime.sh     |  4 +-
 examples/kubernetes_example/src/main.rs       | 46 ++++++++++++++++++-
 src/lib.rs                                    |  6 +--
 turbolift_internals/src/kubernetes.rs         |  7 +++
 6 files changed, 65 insertions(+), 7 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 63091ce2..72622787 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -8,7 +8,7 @@ readme = "README.md"
 license = "Hippocratic-2.1"
 
 [features]
-distributed = ["chrono", "cached", "async-std", "turbolift_macros/distributed"]
+distributed = ["chrono", "async-std", "turbolift_macros/distributed"]
 service = ["serde_json"]
 # todo we can optimize reqs for children with this load
 
@@ -17,6 +17,6 @@ turbolift_macros = { path = "./turbolift_macros" }
 turbolift_internals = { path = "./turbolift_internals" }
 async-std = { version = "1.6", optional = true }
 chrono = { version = "0.4", optional = true }
-cached = { version = "0.19", optional = true }
+cached = { version = "0.19" }
 actix-web = { version = "3" }
 serde_json = { version = "1", optional = true }
\ No newline at end of file
diff --git a/examples/kubernetes_example/Cargo.toml b/examples/kubernetes_example/Cargo.toml
index 5ee7c45a..d6e1afbb 100644
--- a/examples/kubernetes_example/Cargo.toml
+++ b/examples/kubernetes_example/Cargo.toml
@@ -8,4 +8,9 @@ edition = "2018"
 "distributed" = ["turbolift/distributed"]
 
 [dependencies]
+rand = "0.7"
+async-std = "1"
+lazy_static = "1"
+futures = "0.3"
+cute = "0.3"
 turbolift = { path="../../" }
\ No newline at end of file
diff --git a/examples/kubernetes_example/example_runtime.sh b/examples/kubernetes_example/example_runtime.sh
index cb89e8b6..423f63dc 100644
--- a/examples/kubernetes_example/example_runtime.sh
+++ b/examples/kubernetes_example/example_runtime.sh
@@ -12,4 +12,6 @@ RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test --features "distr
 
 # run
 RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run
-RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run --features "distributed"
\ No newline at end of file
+RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run --features "distributed"
+
+../../kind delete cluster
\ No newline at end of file
diff --git a/examples/kubernetes_example/src/main.rs b/examples/kubernetes_example/src/main.rs
index e7a11a96..9b7f9c74 100644
--- a/examples/kubernetes_example/src/main.rs
+++ b/examples/kubernetes_example/src/main.rs
@@ -1,3 +1,47 @@
+use std::sync::Mutex;
+
+#[macro_use]
+extern crate lazy_static;
+#[macro_use(c)]
+extern crate cute;
+use futures::future::try_join_all;
+use rand::{thread_rng, Rng};
+use turbolift::kubernetes::K8s;
+use turbolift::on;
+
+lazy_static! {
+    static ref K8S: Mutex<K8s> = Mutex::new(K8s::new());
+}
+
+#[on(K8S)]
+fn square(u: u64) -> u64 {
+    u * u
+}
+
+fn random_numbers() -> Vec<u64> {
+    let mut pseud = thread_rng();
+    c![pseud.gen_range(0, 1000), for _i in 1..10]
+}
+
 fn main() {
-    println!("Hello, world!");
+    let input = random_numbers();
+    let futures = c![square(*int), for int in &input];
+    let output = async_std::task::block_on(try_join_all(futures)).unwrap();
+    println!("input: {:?}\noutput: {:?}", input, output);
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn it_works() {
+        let input = random_numbers();
+        let futures = c![square(*int), for int in &input];
+        let output = async_std::task::block_on(try_join_all(futures)).unwrap();
+        assert_eq!(
+            output,
+            input.into_iter().map(|x| x * x).collect::<Vec<u64>>()
+        );
+    }
 }
diff --git a/src/lib.rs b/src/lib.rs
index 1253f800..98d1c39c 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,14 +1,14 @@
 #[cfg(feature = "distributed")]
 pub use async_std;
 #[cfg(feature = "distributed")]
-pub use cached;
-#[cfg(feature = "distributed")]
 pub use chrono;
 
-pub use actix_web;
 #[cfg(feature = "service")]
 pub use serde_json;
 
+pub use actix_web;
+pub use cached;
+
 pub use distributed_platform::{DistributionPlatform, DistributionResult};
 pub use turbolift_internals::*;
 pub use turbolift_macros::*;
diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 3dfb0b89..3d3affe0 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -12,10 +12,17 @@ use crate::distributed_platform::{
 const K8S_NAMESPACE: &str = "turbolift";
 type ImageTag = String;
 
+#[derive(Default)]
 pub struct K8s {
     pods: Vec<Pod>,
 }
 
+impl K8s {
+    pub fn new() -> K8s {
+        K8s { pods: Vec::new() }
+    }
+}
+
 #[async_trait]
 impl DistributionPlatform for K8s {
     async fn declare(&mut self, function_name: &str, project_tar: &[u8]) -> DistributionResult<()> {

From daa387078fbd13ff05b9d40daf938934b269ed59 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Wed, 30 Sep 2020 18:41:50 +0200
Subject: [PATCH 008/145] remove unimplemented service feature

---
 Cargo.toml                               | 3 +--
 src/lib.rs                               | 4 +---
 turbolift_internals/src/build_project.rs | 2 +-
 3 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 72622787..490a9539 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -9,7 +9,6 @@ license = "Hippocratic-2.1"
 
 [features]
 distributed = ["chrono", "async-std", "turbolift_macros/distributed"]
-service = ["serde_json"]
 # todo we can optimize reqs for children with this load
 
 [dependencies]
@@ -19,4 +18,4 @@ async-std = { version = "1.6", optional = true }
 chrono = { version = "0.4", optional = true }
 cached = { version = "0.19" }
 actix-web = { version = "3" }
-serde_json = { version = "1", optional = true }
\ No newline at end of file
+serde_json = { version = "1" }
\ No newline at end of file
diff --git a/src/lib.rs b/src/lib.rs
index 98d1c39c..77a32b39 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -3,11 +3,9 @@ pub use async_std;
 #[cfg(feature = "distributed")]
 pub use chrono;
 
-#[cfg(feature = "service")]
-pub use serde_json;
-
 pub use actix_web;
 pub use cached;
+pub use serde_json;
 
 pub use distributed_platform::{DistributionPlatform, DistributionResult};
 pub use turbolift_internals::*;
diff --git a/turbolift_internals/src/build_project.rs b/turbolift_internals/src/build_project.rs
index 2293f336..49c5f4a0 100644
--- a/turbolift_internals/src/build_project.rs
+++ b/turbolift_internals/src/build_project.rs
@@ -102,7 +102,7 @@ pub fn lint(proj_path: &Path) -> anyhow::Result<()> {
 pub fn make_executable(proj_path: &Path, dest: Option<&Path>) -> anyhow::Result<()> {
     let status = Command::new("cargo")
         .current_dir(proj_path)
-        .args("build --release --features \"distributed,service\"".split(' '))
+        .args("build --release --features \"distributed\"".split(' '))
         .status()?;
 
     if !status.success() {

From e2fbf290d55f76348718f9a2f9daf1aa26ae1b96 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Wed, 30 Sep 2020 19:13:13 +0200
Subject: [PATCH 009/145] bug fix: make sure that we only return cachable
 values from cached functions

---
 turbolift_internals/Cargo.toml | 1 -
 turbolift_macros/src/lib.rs    | 4 ++--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/turbolift_internals/Cargo.toml b/turbolift_internals/Cargo.toml
index 19bddf1b..890ba5cf 100644
--- a/turbolift_internals/Cargo.toml
+++ b/turbolift_internals/Cargo.toml
@@ -30,7 +30,6 @@ cached = "0.19"
 async-std = "1.6"
 async-trait = "0.1"
 
-
 # kubernetes-specific requirements
 kube = "0.42.0"
 kube-runtime = "0.42.0"
diff --git a/turbolift_macros/src/lib.rs b/turbolift_macros/src/lib.rs
index 2922e628..be072f99 100644
--- a/turbolift_macros/src/lib.rs
+++ b/turbolift_macros/src/lib.rs
@@ -164,7 +164,7 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
 
             // call .declare once by memoizing the call
             #[cached(size=1)]
-            async fn setup() -> DistributionResult<()> {
+            async fn setup() {
                 #distribution_platform
                     .lock()
                     .unwrap()
@@ -172,7 +172,7 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
                     .await
                     .expect(#declaration_error_text)
             }
-            setup().await?;
+            setup().await;
 
             let params = #params_vec.join("/");
 

From ba9ce1b3a933adcd566126077e83343485a73dd4 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Wed, 30 Sep 2020 19:58:53 +0200
Subject: [PATCH 010/145] don't make services use the distributed feature flag

---
 turbolift_internals/src/build_project.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/turbolift_internals/src/build_project.rs b/turbolift_internals/src/build_project.rs
index 49c5f4a0..308ef177 100644
--- a/turbolift_internals/src/build_project.rs
+++ b/turbolift_internals/src/build_project.rs
@@ -102,7 +102,7 @@ pub fn lint(proj_path: &Path) -> anyhow::Result<()> {
 pub fn make_executable(proj_path: &Path, dest: Option<&Path>) -> anyhow::Result<()> {
     let status = Command::new("cargo")
         .current_dir(proj_path)
-        .args("build --release --features \"distributed\"".split(' '))
+        .args("build --release".split(' '))
         .status()?;
 
     if !status.success() {

From dffe67a4ebaf6ee2e7cdb96855d9e3700e296a5d Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Wed, 30 Sep 2020 21:17:50 +0200
Subject: [PATCH 011/145] bug fix: add missing import

---
 turbolift_macros/Cargo.toml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/turbolift_macros/Cargo.toml b/turbolift_macros/Cargo.toml
index c693e1d6..6160f5fe 100644
--- a/turbolift_macros/Cargo.toml
+++ b/turbolift_macros/Cargo.toml
@@ -17,4 +17,5 @@ quote = "1"
 proc-macro2 = "1"
 fs_extra = "1"
 turbolift_internals = { path="../turbolift_internals" }
-futures = "0.3"
\ No newline at end of file
+futures = "0.3"
+cached = "0.19"
\ No newline at end of file

From d5e844e5041f642c9c28d2584bbf225168c090a1 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Wed, 30 Sep 2020 21:23:57 +0200
Subject: [PATCH 012/145] update kubernetes tests

---
 examples/kubernetes_example/example_runtime.sh | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/examples/kubernetes_example/example_runtime.sh b/examples/kubernetes_example/example_runtime.sh
index 423f63dc..19cf339f 100644
--- a/examples/kubernetes_example/example_runtime.sh
+++ b/examples/kubernetes_example/example_runtime.sh
@@ -2,16 +2,20 @@
 set -e
 
 # setup cluster (will be used in all tests & runs)
-./kind create cluster
-
 cd examples/kubernetes_example
 
-# test
+# run non-distributed example without cluster in environment
 RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test
-RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test --features "distributed"
+RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run
+
+../../kind create cluster
 
-# run
+# run same tests with cluster in environment
+RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test
 RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run
+
+# run distributed tests
+RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test --features "distributed"
 RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run --features "distributed"
 
 ../../kind delete cluster
\ No newline at end of file

From 912e01bcdb17da61cc630aad905de369c1587c37 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Thu, 1 Oct 2020 08:49:52 +0200
Subject: [PATCH 013/145] debug cached import error error[E0433]: failed to
 resolve: could not find `cached` in `{{root}}`   --> src/main.rs:16:1    | 16
 | #[on(K8S)]    | ^^^^^^^^^^ not found in `cached::async_mutex`

---
 turbolift_macros/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/turbolift_macros/src/lib.rs b/turbolift_macros/src/lib.rs
index be072f99..577cc363 100644
--- a/turbolift_macros/src/lib.rs
+++ b/turbolift_macros/src/lib.rs
@@ -160,7 +160,8 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
             use turbolift::DistributionPlatform;
             use turbolift::DistributionResult;
             use turbolift::async_std::task;
-            use turbolift::cached::proc_macro::cached;
+            extern crate turbolift::cached as ::cached; // cached proc_macro needs lib in {{root}}
+            pub use cached::proc_macro::cached;
 
             // call .declare once by memoizing the call
             #[cached(size=1)]

From d5ebdf09960d2f0b313f6a2d66238dc1c2e5b82c Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Mon, 5 Oct 2020 19:03:15 +0200
Subject: [PATCH 014/145] use has_declared instead of memoizing the declaration
 of a function on a distribution platform

---
 .github/workflows/rust.yml                    |  2 +-
 .pre-commit-config.yaml                       |  2 +-
 Cargo.toml                                    |  1 -
 .../kubernetes_example/example_runtime.sh     |  4 ++--
 examples/local_queue_example/Dockerfile       |  4 ++--
 src/lib.rs                                    |  1 -
 .../src/distributed_platform.rs               |  2 ++
 turbolift_internals/src/kubernetes.rs         | 23 +++++++++++--------
 turbolift_internals/src/local_queue.rs        |  4 ++++
 turbolift_macros/src/lib.rs                   | 21 +++++------------
 10 files changed, 32 insertions(+), 32 deletions(-)

diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index cc93f87c..ef78c20a 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -31,6 +31,6 @@ jobs:
     - name: Check without distributed feature
       run: RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo check
     - name: Check with distributed feature
-      run: RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo check --features "distributed"
+      run: RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo check --features distributed
     - name: Clippy
       run: RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo clippy -- -D warnings
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index c7d4ff99..4637ed19 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -25,7 +25,7 @@
         -   id: check
             name: Check
             description: Runs `cargo check` on the repository with distributed flag
-            entry: bash -c 'RUSTFLAGS='"'"'--cfg procmacro2_semver_exempt'"'"' cargo check --features "distributed" "$@"' --
+            entry: bash -c 'RUSTFLAGS='"'"'--cfg procmacro2_semver_exempt'"'"' cargo check --features distributed "$@"' --
             language: system
             types: [ rust ]
             pass_filenames: false
\ No newline at end of file
diff --git a/Cargo.toml b/Cargo.toml
index 490a9539..d4e1b3f9 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -16,6 +16,5 @@ turbolift_macros = { path = "./turbolift_macros" }
 turbolift_internals = { path = "./turbolift_internals" }
 async-std = { version = "1.6", optional = true }
 chrono = { version = "0.4", optional = true }
-cached = { version = "0.19" }
 actix-web = { version = "3" }
 serde_json = { version = "1" }
\ No newline at end of file
diff --git a/examples/kubernetes_example/example_runtime.sh b/examples/kubernetes_example/example_runtime.sh
index 19cf339f..9401023f 100644
--- a/examples/kubernetes_example/example_runtime.sh
+++ b/examples/kubernetes_example/example_runtime.sh
@@ -15,7 +15,7 @@ RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test
 RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run
 
 # run distributed tests
-RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test --features "distributed"
-RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run --features "distributed"
+RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test --features distributed
+RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run --features distributed
 
 ../../kind delete cluster
\ No newline at end of file
diff --git a/examples/local_queue_example/Dockerfile b/examples/local_queue_example/Dockerfile
index be9a5c9b..a18c40a3 100644
--- a/examples/local_queue_example/Dockerfile
+++ b/examples/local_queue_example/Dockerfile
@@ -5,8 +5,8 @@ WORKDIR examples/local_queue_example
 
 # test
 RUN RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test
-RUN RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test --features "distributed"
+RUN RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test --features distributed
 
 # run
 RUN RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run
-RUN RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run --features "distributed"
\ No newline at end of file
+RUN RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run --features distributed
\ No newline at end of file
diff --git a/src/lib.rs b/src/lib.rs
index 77a32b39..4ee38be5 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -4,7 +4,6 @@ pub use async_std;
 pub use chrono;
 
 pub use actix_web;
-pub use cached;
 pub use serde_json;
 
 pub use distributed_platform::{DistributionPlatform, DistributionResult};
diff --git a/turbolift_internals/src/distributed_platform.rs b/turbolift_internals/src/distributed_platform.rs
index 2a8a36c1..ba0bfce7 100644
--- a/turbolift_internals/src/distributed_platform.rs
+++ b/turbolift_internals/src/distributed_platform.rs
@@ -19,4 +19,6 @@ pub trait DistributionPlatform {
         function_name: &str,
         params: ArgsString,
     ) -> DistributionResult<JsonResponse>;
+
+    fn has_declared(&self, fn_name: &str) -> bool;
 }
diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 3d3affe0..3b083749 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -1,8 +1,8 @@
 use async_trait::async_trait;
-use cached::proc_macro::cached;
 use k8s_openapi::api::core::v1::Pod;
 use kube::api::{Api, PostParams};
 use kube::Client;
+use std::collections::HashMap;
 use url::Url;
 
 use crate::distributed_platform::{
@@ -14,12 +14,12 @@ type ImageTag = String;
 
 #[derive(Default)]
 pub struct K8s {
-    pods: Vec<Pod>,
+    fn_names_to_pods: HashMap<String, Pod>,
 }
 
 impl K8s {
     pub fn new() -> K8s {
-        K8s { pods: Vec::new() }
+        Default::default()
     }
 }
 
@@ -31,7 +31,7 @@ impl DistributionPlatform for K8s {
         let pods: Api<Pod> = Api::namespaced(client, K8S_NAMESPACE);
 
         // generate image & host it on a local repo
-        let repo_url = setup_repo();
+        let repo_url = setup_repo(function_name, project_tar)?;
         let local_tag = make_image(function_name, project_tar)?;
         let tag_in_repo = add_image_to_repo(local_tag)?;
         let image_url = repo_url.join(&tag_in_repo)?;
@@ -54,9 +54,11 @@ impl DistributionPlatform for K8s {
                 ],
             }
         }))?;
-        self.pods
-            .push(pods.create(&PostParams::default(), &pod).await?);
-        // todo do we need to monitor the pod in any way??
+        self.fn_names_to_pods.insert(
+            function_name.to_string(),
+            pods.create(&PostParams::default(), &pod).await?,
+        );
+        // todo we should make sure that the pod is accepted, and should make sure it didn't error
         Ok(())
     }
 
@@ -67,10 +69,13 @@ impl DistributionPlatform for K8s {
     ) -> DistributionResult<JsonResponse> {
         unimplemented!()
     }
+
+    fn has_declared(&self, fn_name: &str) -> bool {
+        self.fn_names_to_pods.contains_key(fn_name)
+    }
 }
 
-#[cached(size = 1)]
-fn setup_repo() -> Url {
+fn setup_repo(_function_name: &str, _project_tar: &[u8]) -> DistributionResult<Url> {
     unimplemented!()
 }
 
diff --git a/turbolift_internals/src/local_queue.rs b/turbolift_internals/src/local_queue.rs
index 712293c6..00130db3 100644
--- a/turbolift_internals/src/local_queue.rs
+++ b/turbolift_internals/src/local_queue.rs
@@ -95,6 +95,10 @@ impl DistributionPlatform for LocalQueue {
         // ^ todo not sure why futures are hanging here unless I wrap them in a new block_on?
         Ok(response)
     }
+
+    fn has_declared(&self, fn_name: &str) -> bool {
+        self.fn_name_to_binary_path.contains_key(fn_name)
+    }
 }
 
 impl Drop for LocalQueue {
diff --git a/turbolift_macros/src/lib.rs b/turbolift_macros/src/lib.rs
index 577cc363..62cccb9e 100644
--- a/turbolift_macros/src/lib.rs
+++ b/turbolift_macros/src/lib.rs
@@ -40,10 +40,6 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
         &function_name_string,
         untyped_params,
     );
-    let declaration_error_text = format!(
-        "turbolift: error while declaring {}",
-        original_target_function_name
-    );
 
     // todo extract any docs from passed function and put into fn wrapper
 
@@ -157,23 +153,18 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
         async fn #original_target_function_ident(#typed_params) ->
             turbolift::DistributionResult<#result_type> {
             use std::time::Duration;
-            use turbolift::DistributionPlatform;
+            use turbolift::distributed_platform::DistributionPlatform;
             use turbolift::DistributionResult;
             use turbolift::async_std::task;
-            extern crate turbolift::cached as ::cached; // cached proc_macro needs lib in {{root}}
-            pub use cached::proc_macro::cached;
 
-            // call .declare once by memoizing the call
-            #[cached(size=1)]
-            async fn setup() {
+            if !#distribution_platform
+                .lock()?
+                .has_declared(#original_target_function_name) {
                 #distribution_platform
-                    .lock()
-                    .unwrap()
+                    .lock()?
                     .declare(#original_target_function_name, #project_source_binary)
-                    .await
-                    .expect(#declaration_error_text)
+                    .await?;
             }
-            setup().await;
 
             let params = #params_vec.join("/");
 

From 281f7324e935a7aefc60c90f97b4e19fe0bc1a15 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Mon, 5 Oct 2020 19:12:53 +0200
Subject: [PATCH 015/145] lock the distribution_platform mutex once instead of
 thrice while dispatching a function

---
 turbolift_macros/src/lib.rs | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/turbolift_macros/src/lib.rs b/turbolift_macros/src/lib.rs
index 62cccb9e..1de8cfda 100644
--- a/turbolift_macros/src/lib.rs
+++ b/turbolift_macros/src/lib.rs
@@ -157,19 +157,17 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
             use turbolift::DistributionResult;
             use turbolift::async_std::task;
 
-            if !#distribution_platform
-                .lock()?
-                .has_declared(#original_target_function_name) {
-                #distribution_platform
-                    .lock()?
+            let platform = #distribution_platform.lock()?;
+
+            if !platform.has_declared(#original_target_function_name) {
+                platform
                     .declare(#original_target_function_name, #project_source_binary)
                     .await?;
             }
 
             let params = #params_vec.join("/");
 
-            let resp_string = #distribution_platform
-                .lock()?
+            let resp_string = platform
                 .dispatch(
                     #original_target_function_name,
                     params.to_string()

From 950858575eab2361d32c71bdc038e64739c49511 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Tue, 6 Oct 2020 10:23:58 +0200
Subject: [PATCH 016/145] make platform mutable

---
 turbolift_macros/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/turbolift_macros/src/lib.rs b/turbolift_macros/src/lib.rs
index 1de8cfda..969e0bc7 100644
--- a/turbolift_macros/src/lib.rs
+++ b/turbolift_macros/src/lib.rs
@@ -157,7 +157,7 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
             use turbolift::DistributionResult;
             use turbolift::async_std::task;
 
-            let platform = #distribution_platform.lock()?;
+            let mut platform = #distribution_platform.lock()?;
 
             if !platform.has_declared(#original_target_function_name) {
                 platform

From 61a103cfd9ba5efea178c3859e5b36fe1663d8e5 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Tue, 6 Oct 2020 14:05:48 +0200
Subject: [PATCH 017/145] implement setup_repo for k8s

---
 README.md                             |  5 +++
 turbolift_internals/Cargo.toml        |  2 ++
 turbolift_internals/src/kubernetes.rs | 49 ++++++++++++++++++++++++---
 3 files changed, 51 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index b417e3c0..afcd7c5a 100644
--- a/README.md
+++ b/README.md
@@ -52,6 +52,11 @@ should not have relative local dependencies prone to breaking.
 - if your program produces side effects when initialized, for example when 
 global constants are initialized, those side effects may be triggered 
 for each function call.
+- currently, turbolift does not guarantee that the target operating system for a 
+program will also be used with its microservices.
+- currently, turbolift's k8s support only works with the en0 interface. For devices 
+with multiple network interfaces (en1, en2, ...), we ignore all other interfaces
+while sharing information from the across the local network.
 
 ## Current Project Goals
 - [ ] support kubernetes ([pr](https://github.com/DominicBurkart/turbolift/pull/2)).
diff --git a/turbolift_internals/Cargo.toml b/turbolift_internals/Cargo.toml
index 890ba5cf..377debdb 100644
--- a/turbolift_internals/Cargo.toml
+++ b/turbolift_internals/Cargo.toml
@@ -29,6 +29,8 @@ anyhow = "1"
 cached = "0.19"
 async-std = "1.6"
 async-trait = "0.1"
+base64 = "0.13"
+get_if_addrs = "0.5.3"
 
 # kubernetes-specific requirements
 kube = "0.42.0"
diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 3b083749..0f6a884c 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -1,8 +1,12 @@
+use std::collections::HashMap;
+use std::process::Command;
+use std::str::FromStr;
+
 use async_trait::async_trait;
+use base64::encode;
 use k8s_openapi::api::core::v1::Pod;
 use kube::api::{Api, PostParams};
 use kube::Client;
-use std::collections::HashMap;
 use url::Url;
 
 use crate::distributed_platform::{
@@ -75,12 +79,47 @@ impl DistributionPlatform for K8s {
     }
 }
 
-fn setup_repo(_function_name: &str, _project_tar: &[u8]) -> DistributionResult<Url> {
-    unimplemented!()
+fn setup_repo(_function_name: &str, _project_tar: &[u8]) -> anyhow::Result<Url> {
+    let status = Command::new("docker")
+        .args("run -d -p 5000:5000 --restart=always --name registry registry:2".split(' '))
+        .status()?; // todo choose an open port
+    if !status.success() {
+        return Err(anyhow::anyhow!("repo setup failed"));
+    }
+    let interfaces = get_if_addrs::get_if_addrs()?;
+    for interface in interfaces {
+        if interface.name == "en0" {
+            // todo support other network interfaces
+            let ip = interface.addr.ip();
+            return Ok(Url::from_str(&(ip.to_string() + ":5000"))?);
+        }
+    }
+    Err(anyhow::anyhow!("no en0 interface found"))
 }
 
-fn make_image(_function_name: &str, _project_tar: &[u8]) -> DistributionResult<ImageTag> {
-    unimplemented!()
+fn make_image(function_name: &str, project_tar: &[u8]) -> anyhow::Result<ImageTag> {
+    let tar_base64 = encode(project_tar);
+    let mut docker_file = format!(
+        "\
+FROM rustlang/rust:nightly
+RUN  apt-get update \
+  && apt-get install -y coreutils \
+  && rm -rf /var/lib/apt/lists/*
+base64 --decode {} > f.tar
+tar xvf f.tar
+",
+        tar_base64
+    );
+    docker_file.insert(0, '\'');
+    docker_file.push('\'');
+    let tag_flag = "-t ".to_string() + function_name;
+    let status = Command::new("docker")
+        .args(&["build", &tag_flag, "-", &docker_file])
+        .status()?;
+    if status.success() {
+        return Err(anyhow::anyhow!("docker image build failure"));
+    }
+    Ok(function_name.to_string())
 }
 
 fn add_image_to_repo(_local_tag: ImageTag) -> DistributionResult<ImageTag> {

From 9af4be250f069233beb92687541bbbcecafb75e1 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Wed, 7 Oct 2020 12:21:12 +0200
Subject: [PATCH 018/145] implement make_image for k8s

---
 README.md                             |   4 +
 turbolift_internals/Cargo.toml        |   2 +-
 turbolift_internals/src/kubernetes.rs | 105 ++++++++++++++++++--------
 3 files changed, 80 insertions(+), 31 deletions(-)

diff --git a/README.md b/README.md
index afcd7c5a..4d85af9a 100644
--- a/README.md
+++ b/README.md
@@ -57,6 +57,10 @@ program will also be used with its microservices.
 - currently, turbolift's k8s support only works with the en0 interface. For devices 
 with multiple network interfaces (en1, en2, ...), we ignore all other interfaces
 while sharing information from the across the local network.
+- When running k8s cluster, a local registry is set up on the host machine 
+by using the registry image. Distribution is handled by this registry, which currently 
+does not handle auto-deletion of distributed images. This means that the image must occasionally 
+be wiped if the same environment is reused to distribute many functions over time.
 
 ## Current Project Goals
 - [ ] support kubernetes ([pr](https://github.com/DominicBurkart/turbolift/pull/2)).
diff --git a/turbolift_internals/Cargo.toml b/turbolift_internals/Cargo.toml
index 377debdb..a3c4c75e 100644
--- a/turbolift_internals/Cargo.toml
+++ b/turbolift_internals/Cargo.toml
@@ -29,8 +29,8 @@ anyhow = "1"
 cached = "0.19"
 async-std = "1.6"
 async-trait = "0.1"
-base64 = "0.13"
 get_if_addrs = "0.5.3"
+regex = "1"
 
 # kubernetes-specific requirements
 kube = "0.42.0"
diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 0f6a884c..3eb0db07 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -3,10 +3,10 @@ use std::process::Command;
 use std::str::FromStr;
 
 use async_trait::async_trait;
-use base64::encode;
 use k8s_openapi::api::core::v1::Pod;
 use kube::api::{Api, PostParams};
 use kube::Client;
+use regex::Regex;
 use url::Url;
 
 use crate::distributed_platform::{
@@ -79,47 +79,92 @@ impl DistributionPlatform for K8s {
     }
 }
 
+lazy_static! {
+    static ref PORT_RE: Regex = Regex::new(r"0\.0\.0\.0:(\d+)->").unwrap();
+}
+
 fn setup_repo(_function_name: &str, _project_tar: &[u8]) -> anyhow::Result<Url> {
-    let status = Command::new("docker")
-        .args("run -d -p 5000:5000 --restart=always --name registry registry:2".split(' '))
-        .status()?; // todo choose an open port
-    if !status.success() {
-        return Err(anyhow::anyhow!("repo setup failed"));
+    // check if registry is already running locally
+    let ps_output = Command::new("docker")
+        .args("ps -f name=registry".split(' '))
+        .output()?;
+    if !ps_output.status.success() {
+        return Err(anyhow::anyhow!(
+            "docker ps failed. Is the docker daemon running?"
+        ));
     }
+    let utf8 = String::from_utf8(ps_output.stdout)?;
+    let num_lines = utf8.as_str().lines().count();
+    let port = if num_lines == 1 {
+        // registry not running. Start the registry.
+        let status = Command::new("docker")
+            .args("run -d -p 5000:5000 --restart=always --name registry registry:2".split(' '))
+            .status()?; // todo choose an open port instead of just hoping 5000 is open
+        if !status.success() {
+            return Err(anyhow::anyhow!("repo setup failed"));
+        }
+        "5000"
+    } else {
+        // registry is running. Return for already-running registry.
+        PORT_RE.captures(&utf8).unwrap().get(1).unwrap().as_str()
+    };
+
+    // return local ip + the registry port
     let interfaces = get_if_addrs::get_if_addrs()?;
-    for interface in interfaces {
+    for interface in &interfaces {
         if interface.name == "en0" {
             // todo support other network interfaces
             let ip = interface.addr.ip();
-            return Ok(Url::from_str(&(ip.to_string() + ":5000"))?);
+            let ip_and_port = "http://".to_string() + &ip.to_string() + ":" + port;
+            return Ok(Url::from_str(&ip_and_port)?);
         }
     }
-    Err(anyhow::anyhow!("no en0 interface found"))
+    Err(anyhow::anyhow!(
+        "no en0 interface found. interfaces: {:?}",
+        interfaces
+    ))
 }
 
 fn make_image(function_name: &str, project_tar: &[u8]) -> anyhow::Result<ImageTag> {
-    let tar_base64 = encode(project_tar);
-    let mut docker_file = format!(
-        "\
-FROM rustlang/rust:nightly
-RUN  apt-get update \
-  && apt-get install -y coreutils \
-  && rm -rf /var/lib/apt/lists/*
-base64 --decode {} > f.tar
-tar xvf f.tar
-",
-        tar_base64
+    println!("making image");
+    // set up directory and dockerfile
+    let build_dir = std::path::PathBuf::from(format!("{}_k8s_temp_dir", function_name));
+    std::fs::create_dir_all(&build_dir)?;
+    let build_dir_canonical = build_dir.canonicalize()?;
+    let dockerfile_path = build_dir_canonical.join("Dockerfile");
+    let tar_file_name = "source.tar";
+    let tar_path = build_dir_canonical.join(tar_file_name);
+    let docker_file = format!(
+        "FROM rustlang/rust:nightly
+COPY {} {}
+RUN cat {} | tar xvf -
+WORKDIR {}
+ENTRYPOINT [\"cargo\", \"run\"]",
+        tar_file_name, tar_file_name, tar_file_name, function_name
     );
-    docker_file.insert(0, '\'');
-    docker_file.push('\'');
-    let tag_flag = "-t ".to_string() + function_name;
-    let status = Command::new("docker")
-        .args(&["build", &tag_flag, "-", &docker_file])
-        .status()?;
-    if status.success() {
-        return Err(anyhow::anyhow!("docker image build failure"));
-    }
-    Ok(function_name.to_string())
+    std::fs::write(&dockerfile_path, docker_file)?;
+    std::fs::write(&tar_path, project_tar)?;
+
+    let result = (|| {
+        // build image
+        let build_cmd = format!(
+            "build -t {} {}",
+            function_name,
+            build_dir_canonical.to_string_lossy()
+        );
+        let status = Command::new("docker")
+            .args(build_cmd.as_str().split(' '))
+            .status()?;
+
+        // make sure that build completed successfully
+        if !status.success() {
+            return Err(anyhow::anyhow!("docker image build failure"));
+        }
+        Ok(function_name.to_string())
+    })();
+    // always remove the build directory, even on build error
+    std::fs::remove_dir_all(build_dir_canonical)?;
+    result
 }
 
 fn add_image_to_repo(_local_tag: ImageTag) -> DistributionResult<ImageTag> {

From 8e7ded34145183fbae5d035864a6e6e0979c61be Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Wed, 7 Oct 2020 20:27:46 +0200
Subject: [PATCH 019/145] set up autoscale

---
 examples/kubernetes_example/src/main.rs |   2 +-
 turbolift_internals/src/kubernetes.rs   | 150 ++++++++++++++++++++----
 2 files changed, 128 insertions(+), 24 deletions(-)

diff --git a/examples/kubernetes_example/src/main.rs b/examples/kubernetes_example/src/main.rs
index 9b7f9c74..98409ac8 100644
--- a/examples/kubernetes_example/src/main.rs
+++ b/examples/kubernetes_example/src/main.rs
@@ -10,7 +10,7 @@ use turbolift::kubernetes::K8s;
 use turbolift::on;
 
 lazy_static! {
-    static ref K8S: Mutex<K8s> = Mutex::new(K8s::new());
+    static ref K8S: Mutex<K8s> = Mutex::new(K8s::with_max_replicas(2));
 }
 
 #[on(K8S)]
diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 3eb0db07..77b28e24 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -3,7 +3,8 @@ use std::process::Command;
 use std::str::FromStr;
 
 use async_trait::async_trait;
-use k8s_openapi::api::core::v1::Pod;
+use k8s_openapi::api::apps::v1::Deployment;
+use k8s_openapi::api::core::v1::Service;
 use kube::api::{Api, PostParams};
 use kube::Client;
 use regex::Regex;
@@ -12,18 +13,47 @@ use url::Url;
 use crate::distributed_platform::{
     ArgsString, DistributionPlatform, DistributionResult, JsonResponse,
 };
+use syn::export::Formatter;
 
 const K8S_NAMESPACE: &str = "turbolift";
 type ImageTag = String;
+type ServiceAddress = String;
 
-#[derive(Default)]
 pub struct K8s {
-    fn_names_to_pods: HashMap<String, Pod>,
+    max_scale_n: u32,
+    fn_names_to_services: HashMap<String, ServiceAddress>,
 }
 
 impl K8s {
+    /// returns a k8s object that does not perform autoscaling.
     pub fn new() -> K8s {
-        Default::default()
+        K8s::with_max_replicas(1)
+    }
+
+    pub fn with_max_replicas(max: u32) -> K8s {
+        K8s {
+            max_scale_n: max,
+            fn_names_to_services: HashMap::new(),
+        }
+    }
+}
+
+impl Default for K8s {
+    fn default() -> Self {
+        K8s::new()
+    }
+}
+
+pub struct AutoscaleError {}
+impl std::error::Error for AutoscaleError {}
+impl std::fmt::Debug for AutoscaleError {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(f, "error while applying autoscale")
+    }
+}
+impl std::fmt::Display for AutoscaleError {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(f, "error while applying autoscale")
     }
 }
 
@@ -31,8 +61,10 @@ impl K8s {
 impl DistributionPlatform for K8s {
     async fn declare(&mut self, function_name: &str, project_tar: &[u8]) -> DistributionResult<()> {
         // connect to cluster. tries in-cluster configuration first, then falls back to kubeconfig file.
-        let client = Client::try_default().await?;
-        let pods: Api<Pod> = Api::namespaced(client, K8S_NAMESPACE);
+        let deployment_client = Client::try_default().await?;
+        let deployments: Api<Deployment> = Api::namespaced(deployment_client, K8S_NAMESPACE);
+        let service_client = Client::try_default().await?;
+        let services: Api<Service> = Api::namespaced(service_client, K8S_NAMESPACE);
 
         // generate image & host it on a local repo
         let repo_url = setup_repo(function_name, project_tar)?;
@@ -40,29 +72,93 @@ impl DistributionPlatform for K8s {
         let tag_in_repo = add_image_to_repo(local_tag)?;
         let image_url = repo_url.join(&tag_in_repo)?;
 
-        // make pod
-        let pod_name = function_name;
-        let container_name = function_name.to_string() + "-container";
-        let pod = serde_json::from_value(serde_json::json!({
-            "apiVersion": "v1",
-            "kind": "Pod",
+        // make deployment
+        let deployment_name = function_name.to_string() + "-deployment";
+        let service_name = function_name.to_string() + "-service";
+        let app_name = function_name.to_string();
+        let deployment = serde_json::from_value(serde_json::json!({
+            "apiVersion": "apps/v1",
+            "kind": "Deployment",
             "metadata": {
-                "name": pod_name
+                "name": deployment_name,
+                "labels": {
+                    "app": app_name
+                }
             },
             "spec": {
-                "containers": [
-                    {
-                        "name": container_name,
-                        "image": image_url.as_str(),
+                "replicas": 1,
+                "selector": {
+                    "matchLabels": {
+                        "app": app_name
+                    }
+                },
+                "template": {
+                    "metadata": {
+                        "labels": {
+                            "app": app_name
+                        }
                     },
-                ],
+                    "spec": {
+                        "containers": [
+                            {
+                                "name": tag_in_repo,
+                                "image": image_url.as_str(),
+                                "ports": {
+                                    "containerPort": 5000
+                                }
+                            },
+                        ]
+                    }
+                }
+
             }
         }))?;
-        self.fn_names_to_pods.insert(
+        deployments
+            .create(&PostParams::default(), &deployment)
+            .await?;
+
+        // make service pointing to deployment
+        let service = serde_json::from_value(serde_json::json!({
+            "apiVersion": "v1",
+            "kind": "Service",
+            "metadata": {
+                "name": service_name
+            },
+            "spec": {
+                "selector": {
+                    "app": deployment_name
+                },
+                "ports": {
+                    "protocol": "HTTP",
+                    "port": 5000,
+                    "targetPort": 5000
+                }
+            }
+        }))?;
+        services.create(&PostParams::default(), &service).await?;
+
+        // todo make sure that the pod and service were correctly started before returning
+
+        if self.max_scale_n > 1 {
+            // set autoscale
+            let scale_args = format!(
+                "autoscale deployment {} --max={}",
+                deployment_name, self.max_scale_n
+            );
+            let scale_status = Command::new("kubectl")
+                .args(scale_args.as_str().split(' '))
+                .status()?;
+            if !scale_status.success() {
+                return Err(Box::new(AutoscaleError {}));
+                // ^ todo attach error context from child
+            }
+        }
+
+        self.fn_names_to_services.insert(
             function_name.to_string(),
-            pods.create(&PostParams::default(), &pod).await?,
+            service_name + "." + K8S_NAMESPACE, // assume that we have a dns resolver
         );
-        // todo we should make sure that the pod is accepted, and should make sure it didn't error
+        // todo handle deleting the relevant service and deployment for each distributed function.
         Ok(())
     }
 
@@ -75,7 +171,7 @@ impl DistributionPlatform for K8s {
     }
 
     fn has_declared(&self, fn_name: &str) -> bool {
-        self.fn_names_to_pods.contains_key(fn_name)
+        self.fn_names_to_services.contains_key(fn_name)
     }
 }
 
@@ -106,6 +202,7 @@ fn setup_repo(_function_name: &str, _project_tar: &[u8]) -> anyhow::Result<Url>
         "5000"
     } else {
         // registry is running. Return for already-running registry.
+        // todo we should always start a new registry, just with a new port
         PORT_RE.captures(&utf8).unwrap().get(1).unwrap().as_str()
     };
 
@@ -139,7 +236,7 @@ fn make_image(function_name: &str, project_tar: &[u8]) -> anyhow::Result<ImageTa
 COPY {} {}
 RUN cat {} | tar xvf -
 WORKDIR {}
-ENTRYPOINT [\"cargo\", \"run\"]",
+ENTRYPOINT [\"cargo\", \"run\", \"127.0.0.1:5000\"]",
         tar_file_name, tar_file_name, tar_file_name, function_name
     );
     std::fs::write(&dockerfile_path, docker_file)?;
@@ -170,3 +267,10 @@ ENTRYPOINT [\"cargo\", \"run\"]",
 fn add_image_to_repo(_local_tag: ImageTag) -> DistributionResult<ImageTag> {
     unimplemented!()
 }
+
+impl Drop for K8s {
+    fn drop(&mut self) {
+        // we need to empty out the local registry to avoid build up and potential collisions.
+        unimplemented!()
+    }
+}

From cdae75d83e7ce567638ddb175403a748507d4d8c Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Wed, 7 Oct 2020 20:43:18 +0200
Subject: [PATCH 020/145] take en0 or eth0 interface (whichever comes first in
 interface list)

---
 turbolift_internals/src/kubernetes.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 77b28e24..1c9b474b 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -209,8 +209,8 @@ fn setup_repo(_function_name: &str, _project_tar: &[u8]) -> anyhow::Result<Url>
     // return local ip + the registry port
     let interfaces = get_if_addrs::get_if_addrs()?;
     for interface in &interfaces {
-        if interface.name == "en0" {
-            // todo support other network interfaces
+        if (interface.name == "en0") || (interface.name == "eth0") {
+            // todo support other network interfaces and figure out a better way to choose the interface
             let ip = interface.addr.ip();
             let ip_and_port = "http://".to_string() + &ip.to_string() + ":" + port;
             return Ok(Url::from_str(&ip_and_port)?);

From b5c5ba170b7ee312d1216b8c519a0955f5d20585 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Thu, 8 Oct 2020 10:08:48 +0200
Subject: [PATCH 021/145] update readme

---
 README.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 4d85af9a..c5d21736 100644
--- a/README.md
+++ b/README.md
@@ -54,8 +54,9 @@ global constants are initialized, those side effects may be triggered
 for each function call.
 - currently, turbolift does not guarantee that the target operating system for a 
 program will also be used with its microservices.
-- currently, turbolift's k8s support only works with the en0 interface. For devices 
-with multiple network interfaces (en1, en2, ...), we ignore all other interfaces
+- currently, turbolift's k8s support only works with the en0 or eth0 interface. For devices 
+with multiple network interfaces (en1, en2, ..., eth1, eth2, ..., and other kinds of network 
+interfaces), we choose the first en0 or eth0 interface and ignore all other interfaces
 while sharing information from the across the local network.
 - When running k8s cluster, a local registry is set up on the host machine 
 by using the registry image. Distribution is handled by this registry, which currently 

From a839e358f5d5115f7c860a80fe0195ba8288dbc6 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Thu, 8 Oct 2020 11:12:15 +0200
Subject: [PATCH 022/145] impl drop for k8s

---
 README.md                             |  7 +-
 turbolift_internals/src/kubernetes.rs | 92 ++++++++++++++++++++-------
 2 files changed, 76 insertions(+), 23 deletions(-)

diff --git a/README.md b/README.md
index c5d21736..b80d3559 100644
--- a/README.md
+++ b/README.md
@@ -57,11 +57,16 @@ program will also be used with its microservices.
 - currently, turbolift's k8s support only works with the en0 or eth0 interface. For devices 
 with multiple network interfaces (en1, en2, ..., eth1, eth2, ..., and other kinds of network 
 interfaces), we choose the first en0 or eth0 interface and ignore all other interfaces
-while sharing information from the across the local network.
+while sharing information from the across the local network. Instead, we should allow users to 
+choose the relevant network, or infer it based on what the k8s instance is using. 
 - When running k8s cluster, a local registry is set up on the host machine 
 by using the registry image. Distribution is handled by this registry, which currently 
 does not handle auto-deletion of distributed images. This means that the image must occasionally 
 be wiped if the same environment is reused to distribute many functions over time.
+- Currently, it is assumed that all nodes are on the same local network. This is important 
+for distributing images for each extracted function without using a private repository. We 
+need private repository support if we want to support k8s instances where all nodes 
+are not on a single, local network.
 
 ## Current Project Goals
 - [ ] support kubernetes ([pr](https://github.com/DominicBurkart/turbolift/pull/2)).
diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 1c9b474b..2a9778b7 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -17,11 +17,10 @@ use syn::export::Formatter;
 
 const K8S_NAMESPACE: &str = "turbolift";
 type ImageTag = String;
-type ServiceAddress = String;
 
 pub struct K8s {
     max_scale_n: u32,
-    fn_names_to_services: HashMap<String, ServiceAddress>,
+    fn_names_to_services: HashMap<String, Url>,
 }
 
 impl K8s {
@@ -57,6 +56,14 @@ impl std::fmt::Display for AutoscaleError {
     }
 }
 
+fn function_to_service_name(function_name: &str) -> String {
+    function_name.to_string() + "-service"
+}
+
+fn function_to_deployment_name(function_name: &str) -> String {
+    function_name.to_string() + "-deployment"
+}
+
 #[async_trait]
 impl DistributionPlatform for K8s {
     async fn declare(&mut self, function_name: &str, project_tar: &[u8]) -> DistributionResult<()> {
@@ -66,15 +73,15 @@ impl DistributionPlatform for K8s {
         let service_client = Client::try_default().await?;
         let services: Api<Service> = Api::namespaced(service_client, K8S_NAMESPACE);
 
-        // generate image & host it on a local repo
-        let repo_url = setup_repo(function_name, project_tar)?;
+        // generate image & host it on a local registry
+        let registry_url = setup_registry(function_name, project_tar)?;
         let local_tag = make_image(function_name, project_tar)?;
-        let tag_in_repo = add_image_to_repo(local_tag)?;
-        let image_url = repo_url.join(&tag_in_repo)?;
+        let tag_in_reg = add_image_to_registry(local_tag)?;
+        let image_url = registry_url.join(&tag_in_reg)?;
 
         // make deployment
-        let deployment_name = function_name.to_string() + "-deployment";
-        let service_name = function_name.to_string() + "-service";
+        let deployment_name = function_to_deployment_name(function_name);
+        let service_name = function_to_service_name(function_name);
         let app_name = function_name.to_string();
         let deployment = serde_json::from_value(serde_json::json!({
             "apiVersion": "apps/v1",
@@ -101,7 +108,7 @@ impl DistributionPlatform for K8s {
                     "spec": {
                         "containers": [
                             {
-                                "name": tag_in_repo,
+                                "name": tag_in_reg,
                                 "image": image_url.as_str(),
                                 "ports": {
                                     "containerPort": 5000
@@ -135,7 +142,16 @@ impl DistributionPlatform for K8s {
                 }
             }
         }))?;
-        services.create(&PostParams::default(), &service).await?;
+        let service = services.create(&PostParams::default(), &service).await?;
+        let service_ip = format!(
+            "http://{}:5000",
+            service
+                .spec
+                .expect("no specification found for service")
+                .cluster_ip
+                .expect("no cluster ip found for service")
+        );
+        println!("service_ip {}", service_ip);
 
         // todo make sure that the pod and service were correctly started before returning
 
@@ -153,11 +169,8 @@ impl DistributionPlatform for K8s {
                 // ^ todo attach error context from child
             }
         }
-
-        self.fn_names_to_services.insert(
-            function_name.to_string(),
-            service_name + "." + K8S_NAMESPACE, // assume that we have a dns resolver
-        );
+        self.fn_names_to_services
+            .insert(function_name.to_string(), Url::from_str(&service_ip)?);
         // todo handle deleting the relevant service and deployment for each distributed function.
         Ok(())
     }
@@ -179,10 +192,10 @@ lazy_static! {
     static ref PORT_RE: Regex = Regex::new(r"0\.0\.0\.0:(\d+)->").unwrap();
 }
 
-fn setup_repo(_function_name: &str, _project_tar: &[u8]) -> anyhow::Result<Url> {
+fn setup_registry(_function_name: &str, _project_tar: &[u8]) -> anyhow::Result<Url> {
     // check if registry is already running locally
     let ps_output = Command::new("docker")
-        .args("ps -f name=registry".split(' '))
+        .args("ps -f name=turbolift-registry".split(' '))
         .output()?;
     if !ps_output.status.success() {
         return Err(anyhow::anyhow!(
@@ -194,10 +207,13 @@ fn setup_repo(_function_name: &str, _project_tar: &[u8]) -> anyhow::Result<Url>
     let port = if num_lines == 1 {
         // registry not running. Start the registry.
         let status = Command::new("docker")
-            .args("run -d -p 5000:5000 --restart=always --name registry registry:2".split(' '))
+            .args(
+                "run -d -p 5000:5000 --restart=always --name turbolift-registry registry:2"
+                    .split(' '),
+            )
             .status()?; // todo choose an open port instead of just hoping 5000 is open
         if !status.success() {
-            return Err(anyhow::anyhow!("repo setup failed"));
+            return Err(anyhow::anyhow!("registry setup failed"));
         }
         "5000"
     } else {
@@ -264,13 +280,45 @@ ENTRYPOINT [\"cargo\", \"run\", \"127.0.0.1:5000\"]",
     result
 }
 
-fn add_image_to_repo(_local_tag: ImageTag) -> DistributionResult<ImageTag> {
+fn add_image_to_registry(_local_tag: ImageTag) -> DistributionResult<ImageTag> {
     unimplemented!()
 }
 
 impl Drop for K8s {
     fn drop(&mut self) {
-        // we need to empty out the local registry to avoid build up and potential collisions.
-        unimplemented!()
+        // delete the associated services and deployments from the functions we distributed
+        async_std::task::block_on(async {
+            let deployment_client = Client::try_default().await.unwrap();
+            let deployments: Api<Deployment> = Api::namespaced(deployment_client, K8S_NAMESPACE);
+            let service_client = Client::try_default().await.unwrap();
+            let services: Api<Service> = Api::namespaced(service_client, K8S_NAMESPACE);
+
+            let distributed_functions = self.fn_names_to_services.keys();
+            for function in distributed_functions {
+                let service = function_to_service_name(function);
+                services
+                    .delete(&service, &Default::default())
+                    .await
+                    .unwrap();
+                let deployment = function_to_deployment_name(function);
+                deployments
+                    .delete(&deployment, &Default::default())
+                    .await
+                    .unwrap();
+            }
+        });
+
+        // delete the local registry
+        let registry_deletion_status = Command::new("docker")
+            .arg("rmi")
+            .arg("$(docker images |grep 'turbolift-registry')")
+            .status()
+            .unwrap();
+        if !registry_deletion_status.success() {
+            eprintln!(
+                "could not delete turblift registry docker image. error code: {}",
+                registry_deletion_status.code().unwrap()
+            );
+        }
     }
 }

From d87a33d8a3b57468d5056c66ff9bd63d4a1d92ad Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Thu, 8 Oct 2020 11:38:59 +0200
Subject: [PATCH 023/145] get socket assigned from os instead of hoping 5000 is
 open

---
 turbolift_internals/src/kubernetes.rs | 30 +++++++++++++++++----------
 turbolift_internals/src/utils.rs      |  6 ++++++
 2 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 2a9778b7..2b1ac7cb 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -8,12 +8,13 @@ use k8s_openapi::api::core::v1::Service;
 use kube::api::{Api, PostParams};
 use kube::Client;
 use regex::Regex;
+use syn::export::Formatter;
 use url::Url;
 
 use crate::distributed_platform::{
     ArgsString, DistributionPlatform, DistributionResult, JsonResponse,
 };
-use syn::export::Formatter;
+use crate::utils::get_open_socket;
 
 const K8S_NAMESPACE: &str = "turbolift";
 type ImageTag = String;
@@ -206,20 +207,27 @@ fn setup_registry(_function_name: &str, _project_tar: &[u8]) -> anyhow::Result<U
     let num_lines = utf8.as_str().lines().count();
     let port = if num_lines == 1 {
         // registry not running. Start the registry.
+        let port = get_open_socket()?.local_addr()?.port().to_string(); // todo hack
+        let args_str = format!(
+            "run -d -p {}:5000 --restart=always --name turbolift-registry registry:2",
+            port
+        );
         let status = Command::new("docker")
-            .args(
-                "run -d -p 5000:5000 --restart=always --name turbolift-registry registry:2"
-                    .split(' '),
-            )
-            .status()?; // todo choose an open port instead of just hoping 5000 is open
+            .args(args_str.as_str().split(' '))
+            .status()?;
         if !status.success() {
             return Err(anyhow::anyhow!("registry setup failed"));
         }
-        "5000"
+        port
     } else {
-        // registry is running. Return for already-running registry.
-        // todo we should always start a new registry, just with a new port
-        PORT_RE.captures(&utf8).unwrap().get(1).unwrap().as_str()
+        // turbolift-registry is running. Return for already-running registry.
+        PORT_RE
+            .captures(&utf8)
+            .unwrap()
+            .get(1)
+            .unwrap()
+            .as_str()
+            .to_string()
     };
 
     // return local ip + the registry port
@@ -228,7 +236,7 @@ fn setup_registry(_function_name: &str, _project_tar: &[u8]) -> anyhow::Result<U
         if (interface.name == "en0") || (interface.name == "eth0") {
             // todo support other network interfaces and figure out a better way to choose the interface
             let ip = interface.addr.ip();
-            let ip_and_port = "http://".to_string() + &ip.to_string() + ":" + port;
+            let ip_and_port = "http://".to_string() + &ip.to_string() + ":" + &port;
             return Ok(Url::from_str(&ip_and_port)?);
         }
     }
diff --git a/turbolift_internals/src/utils.rs b/turbolift_internals/src/utils.rs
index 52e10e44..e9f02cdd 100644
--- a/turbolift_internals/src/utils.rs
+++ b/turbolift_internals/src/utils.rs
@@ -1,5 +1,11 @@
+use std::net::UdpSocket;
+
 #[cfg(target_family = "unix")]
 pub use std::os::unix::fs::symlink as symlink_dir;
 
 #[cfg(target_family = "windows")]
 pub use std::os::windows::fs::symlink_dir;
+
+pub fn get_open_socket() -> std::io::Result<UdpSocket> {
+    UdpSocket::bind("127.0.0.1:0")
+}

From e6badf95c07de3bd85de5183713be5eb9524ea7b Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Thu, 8 Oct 2020 12:06:18 +0200
Subject: [PATCH 024/145] add minimal docs to k8s

---
 turbolift_internals/src/kubernetes.rs | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 2b1ac7cb..931512e7 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -19,18 +19,30 @@ use crate::utils::get_open_socket;
 const K8S_NAMESPACE: &str = "turbolift";
 type ImageTag = String;
 
+/// `K8s` is the interface for turning rust functions into autoscaling microservices
+/// using turbolift. It requires docker and kubernetes / kubectl to already be setup on the
+/// device at runtime.
+///
+/// Access to the kubernetes cluster must be inferrable from the env variables at runtime.
 pub struct K8s {
     max_scale_n: u32,
     fn_names_to_services: HashMap<String, Url>,
 }
 
 impl K8s {
-    /// returns a k8s object that does not perform autoscaling.
+    /// returns a K8s object that does not perform autoscaling.
     pub fn new() -> K8s {
         K8s::with_max_replicas(1)
     }
 
+    /// returns a K8s object. If max is equal to 1, then autoscaling
+    /// is not enabled. Otherwise, autoscale is automatically activated
+    /// with cluster defaults and a max number of replicas *per distributed
+    /// function* of `max`. Panics if `max` < 1.
     pub fn with_max_replicas(max: u32) -> K8s {
+        if max < 1 {
+            panic!("max < 1 while instantiating k8s (value: {})", max)
+        }
         K8s {
             max_scale_n: max,
             fn_names_to_services: HashMap::new(),
@@ -241,7 +253,7 @@ fn setup_registry(_function_name: &str, _project_tar: &[u8]) -> anyhow::Result<U
         }
     }
     Err(anyhow::anyhow!(
-        "no en0 interface found. interfaces: {:?}",
+        "no en0/eth0 interface found. interfaces: {:?}",
         interfaces
     ))
 }

From b85e4e5cd77641888760dd555f3bfbc397f17059 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Thu, 8 Oct 2020 14:22:46 +0200
Subject: [PATCH 025/145] build project in container

---
 turbolift_internals/src/kubernetes.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 931512e7..08b9c9c4 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -272,7 +272,8 @@ fn make_image(function_name: &str, project_tar: &[u8]) -> anyhow::Result<ImageTa
 COPY {} {}
 RUN cat {} | tar xvf -
 WORKDIR {}
-ENTRYPOINT [\"cargo\", \"run\", \"127.0.0.1:5000\"]",
+RUN cargo build --release
+ENTRYPOINT [\"cargo\", \"run\", \"--release\", \"127.0.0.1:5000\"]",
         tar_file_name, tar_file_name, tar_file_name, function_name
     );
     std::fs::write(&dockerfile_path, docker_file)?;

From 789c573199dc4c579b1ce620762293c3b32c9d75 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Thu, 8 Oct 2020 14:31:15 +0200
Subject: [PATCH 026/145] use cmd syntax instead of entrypoint syntax in
 dockerfile

---
 turbolift_internals/src/kubernetes.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 08b9c9c4..30d60c4e 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -273,7 +273,7 @@ COPY {} {}
 RUN cat {} | tar xvf -
 WORKDIR {}
 RUN cargo build --release
-ENTRYPOINT [\"cargo\", \"run\", \"--release\", \"127.0.0.1:5000\"]",
+CMD cargo run --release 127.0.0.1:5000",
         tar_file_name, tar_file_name, tar_file_name, function_name
     );
     std::fs::write(&dockerfile_path, docker_file)?;

From 989be90f19ad05d23223ea089fe263fcb68b80cb Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Thu, 8 Oct 2020 16:08:29 +0200
Subject: [PATCH 027/145] push image to repo in make_image instead of in a
 separate function

---
 turbolift_internals/src/kubernetes.rs | 35 +++++++++++++++++++--------
 1 file changed, 25 insertions(+), 10 deletions(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 30d60c4e..7872acfd 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -88,8 +88,7 @@ impl DistributionPlatform for K8s {
 
         // generate image & host it on a local registry
         let registry_url = setup_registry(function_name, project_tar)?;
-        let local_tag = make_image(function_name, project_tar)?;
-        let tag_in_reg = add_image_to_registry(local_tag)?;
+        let tag_in_reg = make_image(function_name, project_tar, &registry_url)?;
         let image_url = registry_url.join(&tag_in_reg)?;
 
         // make deployment
@@ -258,7 +257,11 @@ fn setup_registry(_function_name: &str, _project_tar: &[u8]) -> anyhow::Result<U
     ))
 }
 
-fn make_image(function_name: &str, project_tar: &[u8]) -> anyhow::Result<ImageTag> {
+fn make_image(
+    function_name: &str,
+    project_tar: &[u8],
+    registry_url: &Url,
+) -> anyhow::Result<ImageTag> {
     println!("making image");
     // set up directory and dockerfile
     let build_dir = std::path::PathBuf::from(format!("{}_k8s_temp_dir", function_name));
@@ -286,25 +289,37 @@ CMD cargo run --release 127.0.0.1:5000",
             function_name,
             build_dir_canonical.to_string_lossy()
         );
-        let status = Command::new("docker")
+        let build_status = Command::new("docker")
             .args(build_cmd.as_str().split(' '))
             .status()?;
 
         // make sure that build completed successfully
-        if !status.success() {
+        if !build_status.success() {
             return Err(anyhow::anyhow!("docker image build failure"));
         }
-        Ok(function_name.to_string())
+
+        // push image to local repo
+        let image_tag = format!(
+            "{}:{}/{}",
+            registry_url.host_str().unwrap(),
+            registry_url.port().unwrap(),
+            function_name
+        );
+        let push_status = Command::new("docker")
+            .arg("push")
+            .arg(&image_tag)
+            .status()?;
+        if !push_status.success() {
+            return Err(anyhow::anyhow!("docker image push failure"));
+        }
+
+        Ok(image_tag)
     })();
     // always remove the build directory, even on build error
     std::fs::remove_dir_all(build_dir_canonical)?;
     result
 }
 
-fn add_image_to_registry(_local_tag: ImageTag) -> DistributionResult<ImageTag> {
-    unimplemented!()
-}
-
 impl Drop for K8s {
     fn drop(&mut self) {
         // delete the associated services and deployments from the functions we distributed

From d07ca01f86aa7427881cff9184ebfdb85216b75b Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Thu, 8 Oct 2020 16:45:05 +0200
Subject: [PATCH 028/145] implement dispatch

---
 turbolift_internals/src/kubernetes.rs | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 7872acfd..001667cd 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -189,10 +189,20 @@ impl DistributionPlatform for K8s {
 
     async fn dispatch(
         &mut self,
-        _function_name: &str,
-        _params: ArgsString,
+        function_name: &str,
+        params: ArgsString,
     ) -> DistributionResult<JsonResponse> {
-        unimplemented!()
+        async fn get(query_url: Url) -> String {
+            surf::get(query_url).recv_string().await.unwrap()
+        }
+
+        // request from server
+        let address_and_port = self.fn_names_to_services.get(function_name).unwrap();
+        let prefixed_params = "./".to_string() + function_name + "/" + &params;
+        let query_url = address_and_port.join(&prefixed_params)?;
+        let response = async_std::task::block_on(get(query_url));
+        // ^ todo not sure why futures are hanging here unless I wrap them in a new block_on?
+        Ok(response)
     }
 
     fn has_declared(&self, fn_name: &str) -> bool {

From 6476b2798fa55206a97e6e8084c1288c93551271 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Thu, 8 Oct 2020 17:02:11 +0200
Subject: [PATCH 029/145] update names

---
 turbolift_internals/src/kubernetes.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 001667cd..0d96618d 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -197,9 +197,9 @@ impl DistributionPlatform for K8s {
         }
 
         // request from server
-        let address_and_port = self.fn_names_to_services.get(function_name).unwrap();
-        let prefixed_params = "./".to_string() + function_name + "/" + &params;
-        let query_url = address_and_port.join(&prefixed_params)?;
+        let service_base_url = self.fn_names_to_services.get(function_name).unwrap();
+        let args = "./".to_string() + function_name + "/" + &params;
+        let query_url = service_base_url.join(&args)?;
         let response = async_std::task::block_on(get(query_url));
         // ^ todo not sure why futures are hanging here unless I wrap them in a new block_on?
         Ok(response)

From 730ba0df4894549eb2cddc620ebfaef6de370910 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Thu, 8 Oct 2020 19:15:40 +0200
Subject: [PATCH 030/145] use relative path for local import definitions
 instead of absolute path absolute paths don't make sense in VMs or on other
 filesystems whent the tar is moved & unpacked

---
 turbolift_internals/Cargo.toml           |  1 +
 turbolift_internals/src/build_project.rs | 11 +++++++++--
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/turbolift_internals/Cargo.toml b/turbolift_internals/Cargo.toml
index a3c4c75e..34d5748a 100644
--- a/turbolift_internals/Cargo.toml
+++ b/turbolift_internals/Cargo.toml
@@ -31,6 +31,7 @@ async-std = "1.6"
 async-trait = "0.1"
 get_if_addrs = "0.5.3"
 regex = "1"
+pathdiff = "0.2.0"
 
 # kubernetes-specific requirements
 kube = "0.42.0"
diff --git a/turbolift_internals/src/build_project.rs b/turbolift_internals/src/build_project.rs
index 308ef177..cc105199 100644
--- a/turbolift_internals/src/build_project.rs
+++ b/turbolift_internals/src/build_project.rs
@@ -1,7 +1,10 @@
 use std::collections::HashSet;
 use std::fs;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use std::process::Command;
+use std::str::FromStr;
+
+use pathdiff::diff_paths;
 
 use crate::utils::symlink_dir;
 
@@ -59,7 +62,11 @@ pub fn edit_cargo_file(cargo_path: &Path, function_name: &str) -> anyhow::Result
             } else {
                 symlink_dir(&canonical, &dep_location)?;
             }
-            *buf = dep_location.canonicalize().unwrap();
+
+            let proj_folder = cargo_path.parent().unwrap().canonicalize().unwrap();
+            let rel_dep_location = diff_paths(&dep_location, &proj_folder).unwrap();
+            let relative_path = PathBuf::from_str(".")?.join(&rel_dep_location);
+            *buf = relative_path;
         }
     }
 

From 05178af198dca221885aab6f00ed8e819cf40879 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Thu, 8 Oct 2020 19:18:50 +0200
Subject: [PATCH 031/145] fix misplaced comment

---
 examples/kubernetes_example/example_runtime.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/kubernetes_example/example_runtime.sh b/examples/kubernetes_example/example_runtime.sh
index 9401023f..ce74013e 100644
--- a/examples/kubernetes_example/example_runtime.sh
+++ b/examples/kubernetes_example/example_runtime.sh
@@ -1,13 +1,13 @@
 # error if any command fails
 set -e
 
-# setup cluster (will be used in all tests & runs)
 cd examples/kubernetes_example
 
 # run non-distributed example without cluster in environment
 RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test
 RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run
 
+# setup cluster (will be used in all tests & runs)
 ../../kind create cluster
 
 # run same tests with cluster in environment

From c173215c8ac3c2dc21a3db68aba784712ff7fc35 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Thu, 8 Oct 2020 19:25:54 +0200
Subject: [PATCH 032/145] update readme

---
 README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index b80d3559..7e472c4e 100644
--- a/README.md
+++ b/README.md
@@ -17,7 +17,8 @@ full projects with working syntax examples.
 ## Distribution as an afterthought.
 Turbolift allows developers to turn normal rust functions into distributed services 
  just by tagging them with a macro. This lets you develop in a monorepo environment, 
-but benefit from the scalability of microservice architectures.
+but benefit from the scalability of microservice architectures. Right now, Turbolift 
+only works with K8s, though it's designed to be extended to other cluster management utilities.
 
 ## Orchestration with a feature flag.
 For quicker development builds, `cargo build` doesn't build the distributed version of your code by default.

From bf8b108d10f7b2302a5d03d69f7d33577d3f8f77 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Fri, 9 Oct 2020 14:49:04 +0200
Subject: [PATCH 033/145] always build project and services with
 procmacro2_semver_exempt

---
 turbolift_internals/src/kubernetes.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 0d96618d..4d9bbf00 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -285,8 +285,8 @@ fn make_image(
 COPY {} {}
 RUN cat {} | tar xvf -
 WORKDIR {}
-RUN cargo build --release
-CMD cargo run --release 127.0.0.1:5000",
+RUN RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo build --release
+CMD RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run --release 127.0.0.1:5000",
         tar_file_name, tar_file_name, tar_file_name, function_name
     );
     std::fs::write(&dockerfile_path, docker_file)?;

From b63f691f5327279d6b3981f5458a4086329c4d21 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Fri, 9 Oct 2020 15:03:04 +0200
Subject: [PATCH 034/145] don't hardcode dependency path while building

---
 examples/kubernetes_example/Dockerfile        |  4 +--
 .../kubernetes_example/example_runtime.sh     |  2 +-
 examples/local_queue_example/Dockerfile       |  4 +--
 turbolift_internals/Cargo.toml                |  1 -
 turbolift_internals/src/build_project.rs      | 30 ++++++++-----------
 turbolift_internals/src/extract_function.rs   | 27 ++++++-----------
 turbolift_macros/src/lib.rs                   |  5 ++++
 7 files changed, 32 insertions(+), 41 deletions(-)

diff --git a/examples/kubernetes_example/Dockerfile b/examples/kubernetes_example/Dockerfile
index 6a05bdee..b2540871 100644
--- a/examples/kubernetes_example/Dockerfile
+++ b/examples/kubernetes_example/Dockerfile
@@ -9,5 +9,5 @@ RUN  apt-get update \
   && rm -rf /var/lib/apt/lists/*
 RUN wget -O ./kind --show-progress "https://kind.sigs.k8s.io/dl/v0.9.0/kind-$(uname)-amd64"
 RUN chmod +x ./kind
-COPY ./ ./
-CMD sh examples/kubernetes_example/example_runtime.sh
\ No newline at end of file
+COPY ./ turbolift
+CMD sh turbolift/examples/kubernetes_example/example_runtime.sh
\ No newline at end of file
diff --git a/examples/kubernetes_example/example_runtime.sh b/examples/kubernetes_example/example_runtime.sh
index ce74013e..68f41f3b 100644
--- a/examples/kubernetes_example/example_runtime.sh
+++ b/examples/kubernetes_example/example_runtime.sh
@@ -1,7 +1,7 @@
 # error if any command fails
 set -e
 
-cd examples/kubernetes_example
+cd turbolift/examples/kubernetes_example
 
 # run non-distributed example without cluster in environment
 RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test
diff --git a/examples/local_queue_example/Dockerfile b/examples/local_queue_example/Dockerfile
index a18c40a3..3f792a7d 100644
--- a/examples/local_queue_example/Dockerfile
+++ b/examples/local_queue_example/Dockerfile
@@ -1,7 +1,7 @@
 # run me in turbolift root! E.G.: "docker build -f examples/local_queue_example/Dockerfile ."
 FROM rustlang/rust:nightly
-COPY ./ ./
-WORKDIR examples/local_queue_example
+COPY ./ turbolift
+WORKDIR turbolift/examples/local_queue_example
 
 # test
 RUN RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test
diff --git a/turbolift_internals/Cargo.toml b/turbolift_internals/Cargo.toml
index 34d5748a..a3c4c75e 100644
--- a/turbolift_internals/Cargo.toml
+++ b/turbolift_internals/Cargo.toml
@@ -31,7 +31,6 @@ async-std = "1.6"
 async-trait = "0.1"
 get_if_addrs = "0.5.3"
 regex = "1"
-pathdiff = "0.2.0"
 
 # kubernetes-specific requirements
 kube = "0.42.0"
diff --git a/turbolift_internals/src/build_project.rs b/turbolift_internals/src/build_project.rs
index cc105199..6cb4896b 100644
--- a/turbolift_internals/src/build_project.rs
+++ b/turbolift_internals/src/build_project.rs
@@ -4,14 +4,17 @@ use std::path::{Path, PathBuf};
 use std::process::Command;
 use std::str::FromStr;
 
-use pathdiff::diff_paths;
-
 use crate::utils::symlink_dir;
 
-pub fn edit_cargo_file(cargo_path: &Path, function_name: &str) -> anyhow::Result<()> {
+pub fn edit_cargo_file(
+    original_project_source_dir: &Path,
+    cargo_path: &Path,
+    function_name: &str,
+) -> anyhow::Result<()> {
+    let local_deps_dir_name = ".local_deps";
     let mut parsed_toml: cargo_toml2::CargoToml = cargo_toml2::from_path(cargo_path)
         .unwrap_or_else(|_| panic!("toml at {:?} could not be read", cargo_path));
-    let relative_local_deps_cache = cargo_path.parent().unwrap().join(".local_deps");
+    let relative_local_deps_cache = cargo_path.parent().unwrap().join(local_deps_dir_name);
     fs::create_dir_all(&relative_local_deps_cache)?;
     let local_deps_cache = relative_local_deps_cache.canonicalize()?;
 
@@ -26,21 +29,17 @@ pub fn edit_cargo_file(cargo_path: &Path, function_name: &str) -> anyhow::Result
     let details = deps
         .iter_mut()
         // only full dependency descriptions (not simple version number)
-        .filter_map(|(_name, dep)| match dep {
+        .filter_map(|(name, dep)| match dep {
             cargo_toml2::Dependency::Simple(_) => None,
-            cargo_toml2::Dependency::Full(detail) => Some(detail),
+            cargo_toml2::Dependency::Full(detail) => Some((name, detail)),
         });
     let mut completed_locations = HashSet::new();
-    for detail in details {
+    for (name, detail) in details {
         // only descriptions with a path
         if let Some(ref mut buf) = detail.path {
             // determine what the symlink for this dependency should be
-            let canonical = buf.canonicalize()?;
-            let dep_location = local_deps_cache.join(
-                &canonical
-                    .file_name()
-                    .unwrap_or_else(|| canonical.as_os_str()),
-            );
+            let canonical = original_project_source_dir.join(&buf).canonicalize()?;
+            let dep_location = local_deps_cache.join(name);
 
             // check that we don't have a naming error
             // todo: automatically handle naming conflicts by mangling the dep for one
@@ -63,10 +62,7 @@ pub fn edit_cargo_file(cargo_path: &Path, function_name: &str) -> anyhow::Result
                 symlink_dir(&canonical, &dep_location)?;
             }
 
-            let proj_folder = cargo_path.parent().unwrap().canonicalize().unwrap();
-            let rel_dep_location = diff_paths(&dep_location, &proj_folder).unwrap();
-            let relative_path = PathBuf::from_str(".")?.join(&rel_dep_location);
-            *buf = relative_path;
+            *buf = PathBuf::from_str(".")?.join(local_deps_dir_name).join(name);
         }
     }
 
diff --git a/turbolift_internals/src/extract_function.rs b/turbolift_internals/src/extract_function.rs
index 583bb8cc..0582e185 100644
--- a/turbolift_internals/src/extract_function.rs
+++ b/turbolift_internals/src/extract_function.rs
@@ -1,4 +1,5 @@
 use proc_macro2::TokenStream;
+use std::collections::VecDeque;
 use std::fs;
 use std::io::Cursor;
 use std::path::{Path, PathBuf};
@@ -12,7 +13,6 @@ use syn::spanned::Spanned;
 
 use crate::distributed_platform::DistributionResult;
 use crate::CACHE_PATH;
-use std::collections::VecDeque;
 
 type TypedParams = syn::punctuated::Punctuated<syn::FnArg, syn::Token![,]>;
 type UntypedParams = syn::punctuated::Punctuated<Box<syn::Pat>, syn::Token![,]>;
@@ -166,28 +166,21 @@ pub fn make_compressed_proj_src(dir: &Path) -> Vec<u8> {
         .map(|entry| (dir.file_name().unwrap().into(), entry))
         .collect(); // ignore read errors
 
-    archive.append_dir(dir.file_name().unwrap(), dir).unwrap();
+    let tar_project_base_dir = dir.file_name().unwrap();
+
+    archive.append_dir(tar_project_base_dir, dir).unwrap();
     while !entries.is_empty() {
         let (entry_parent, entry) = entries.pop_front().unwrap();
         if entry.file_name().to_str() == Some("target") && entry.metadata().unwrap().is_dir() {
-            // in target directories, only pass release (if it exists)
-            let release_deps = entry.path().join("release/deps");
-            if release_deps.exists() {
-                let path = {
-                    if entry_parent == dir {
-                        PathBuf::from_str("target/release").unwrap()
-                    } else {
-                        entry_parent.join("target").join("release")
-                    }
-                };
-                archive.append_dir_all(path, release_deps).unwrap();
-            }
+            // ignore target
         } else {
             let entry_path_with_parent = entry_parent.join(entry.file_name());
             if entry.path().is_dir() {
                 // ^ bug: the metadata on symlinks sometimes say that they are not directories,
                 // so we have to metadata.is_dir() || (metadata.file_type().is_symlink() && entry.path().is_dir() )
-                if CACHE_PATH.file_name().unwrap() != entry.file_name() {
+                if CACHE_PATH.file_name().unwrap() == entry.file_name() {
+                    // don't include the cache
+                } else {
                     archive
                         .append_dir(&entry_path_with_parent, entry.path())
                         .unwrap();
@@ -195,10 +188,8 @@ pub fn make_compressed_proj_src(dir: &Path) -> Vec<u8> {
                         fs::read_dir(entry.path())
                             .unwrap()
                             .filter_map(Result::ok)
-                            .map(|child| (entry_path_with_parent.clone(), child)),
+                            .map(|child| (entry_parent.join(entry.file_name()), child)),
                     )
-                } else {
-                    // don't include the cache
                 }
             } else {
                 let mut f = fs::File::open(entry.path()).unwrap();
diff --git a/turbolift_macros/src/lib.rs b/turbolift_macros/src/lib.rs
index 969e0bc7..cbb9828b 100644
--- a/turbolift_macros/src/lib.rs
+++ b/turbolift_macros/src/lib.rs
@@ -109,6 +109,11 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
 
     // modify cargo.toml (edit package info & add actix + json_serde deps)
     build_project::edit_cargo_file(
+        PathBuf::from_str(".")
+            .expect("could not find project dir")
+            .canonicalize()
+            .expect("could not canonicalize path to project dir")
+            .as_path(),
         &function_cache_proj_path.join("Cargo.toml"),
         &original_target_function_name,
     )

From 7355acb7911f552e457bb122786b5f6da90bf1fc Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Wed, 14 Oct 2020 19:15:32 +0200
Subject: [PATCH 035/145] update docs

---
 README.md          |  4 +++-
 examples/README.md | 18 ++++++++++++++++++
 2 files changed, 21 insertions(+), 1 deletion(-)
 create mode 100644 examples/README.md

diff --git a/README.md b/README.md
index 7e472c4e..cac0f7ff 100644
--- a/README.md
+++ b/README.md
@@ -23,7 +23,9 @@ only works with K8s, though it's designed to be extended to other cluster manage
 ## Orchestration with a feature flag.
 For quicker development builds, `cargo build` doesn't build the distributed version of your code by default.
 Distribution is feature-gated so that it's easy to turn on (for example, in production), or off (for example, 
-while developing locally).
+while developing locally). Enabling a dependency feature is as simple as adding a couple of lines to your `Cargo.toml`. 
+Check out the examples directory to see how to add a new feature that triggers turbolift to run in distributed mode
+when activated.
 
 ## Important implementation notes
 - implemented over http using `reqwest` and `actix-web` (no current plans to refactor to use a lower level network protocol).
diff --git a/examples/README.md b/examples/README.md
new file mode 100644
index 00000000..7a68bee2
--- /dev/null
+++ b/examples/README.md
@@ -0,0 +1,18 @@
+# Turbolift Examples
+
+## Kubernetes
+
+Distributing portions of a rust application on Kubernetes was the reason turbolift was made. Internally, turbolift 
+uses the derived source code for each image to create a containerized HTTP server. The container is added to a local 
+ registry, and a deployment and service is then created that exposes the server to requests from the main program. When 
+ the main program completes and the K8s turbolift manager is dropped from memory, it removes the local registry with 
+ the container, as well as the deployment and service. 
+
+## Local Queue
+
+The local queue example should never be used in a production application. It's designed to test the core features of 
+turbolift (automatically extracting microservices from a rust codebase and running them on an http server),
+without any of the platform-specific code for e.g. running on kubernetes. Check this example out if you're interested in 
+a bare-bones example turbolift project without any platform-specific specialization. Note: if you're looking to run code 
+locally in turbolift instead of using a distribution platform, you should deactivate the distributed turbolift feature 
+in your project's `Cargo.toml`. This will let your program run all services locally, e.g. while developing.
\ No newline at end of file

From 7e15b491c00dccfdc0436a6396b8b9b0413e9333 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Wed, 14 Oct 2020 19:17:27 +0200
Subject: [PATCH 036/145] correctly point to kind in k8s example script

---
 examples/kubernetes_example/example_runtime.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/kubernetes_example/example_runtime.sh b/examples/kubernetes_example/example_runtime.sh
index 68f41f3b..0a8f48f7 100644
--- a/examples/kubernetes_example/example_runtime.sh
+++ b/examples/kubernetes_example/example_runtime.sh
@@ -8,7 +8,7 @@ RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test
 RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run
 
 # setup cluster (will be used in all tests & runs)
-../../kind create cluster
+../../../kind create cluster
 
 # run same tests with cluster in environment
 RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test
@@ -18,4 +18,4 @@ RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run
 RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test --features distributed
 RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run --features distributed
 
-../../kind delete cluster
\ No newline at end of file
+../../../kind delete cluster
\ No newline at end of file

From b80bbeecfbc51c81c1cb982449902d45d588cdba Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Wed, 14 Oct 2020 19:48:36 +0200
Subject: [PATCH 037/145] add more hooks to precommit

---
 .pre-commit-config.yaml | 23 ++++++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 4637ed19..349a0746 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,3 +1,4 @@
+repos:
 -    repo: local
      hooks:
         -   id: format
@@ -16,16 +17,32 @@
             types: [rust]
             pass_filenames: false
         -   id: check
-            name: Check
+            name: Cargo Check (local)
             description: Runs `cargo check` on the repository.
             entry: bash -c 'RUSTFLAGS='"'"'--cfg procmacro2_semver_exempt'"'"' cargo check "$@"' --
             language: system
             types: [ rust ]
             pass_filenames: false
         -   id: check
-            name: Check
+            name: Cargo Check (distributed)
             description: Runs `cargo check` on the repository with distributed flag
             entry: bash -c 'RUSTFLAGS='"'"'--cfg procmacro2_semver_exempt'"'"' cargo check --features distributed "$@"' --
             language: system
             types: [ rust ]
-            pass_filenames: false
\ No newline at end of file
+            pass_filenames: false
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v3.2.0
+  hooks:
+    - id: trailing-whitespace
+    - id: mixed-line-ending
+    - id: end-of-file-fixer
+    - id: detect-private-key
+    - id: check-merge-conflict
+    - id: check-toml
+    - id: check-yaml
+- repo: https://github.com/jumanjihouse/pre-commit-hooks
+  rev: 2.1.4
+  hooks:
+    - id: markdownlint
+    - id: shellcheck
+    - id: shfmt

From 7f77614def9d899c1bca1d035bf3de14ba37c1e9 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Fri, 16 Oct 2020 09:45:53 +0200
Subject: [PATCH 038/145] bug fix: tag images to localhost instead of using the
 network address

---
 .pre-commit-config.yaml                       |  1 -
 .../kubernetes_example/example_runtime.sh     |  4 +-
 turbolift_internals/src/kubernetes.rs         | 50 ++++++++++++++-----
 3 files changed, 41 insertions(+), 14 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 349a0746..9f41d618 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -45,4 +45,3 @@ repos:
   hooks:
     - id: markdownlint
     - id: shellcheck
-    - id: shfmt
diff --git a/examples/kubernetes_example/example_runtime.sh b/examples/kubernetes_example/example_runtime.sh
index 0a8f48f7..7e457306 100644
--- a/examples/kubernetes_example/example_runtime.sh
+++ b/examples/kubernetes_example/example_runtime.sh
@@ -1,3 +1,5 @@
+#!/usr/bin/env sh
+
 # error if any command fails
 set -e
 
@@ -18,4 +20,4 @@ RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run
 RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test --features distributed
 RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run --features distributed
 
-../../../kind delete cluster
\ No newline at end of file
+../../../kind delete cluster
diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 4d9bbf00..3a212896 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -89,13 +89,16 @@ impl DistributionPlatform for K8s {
         // generate image & host it on a local registry
         let registry_url = setup_registry(function_name, project_tar)?;
         let tag_in_reg = make_image(function_name, project_tar, &registry_url)?;
-        let image_url = registry_url.join(&tag_in_reg)?;
+        let image_url = registry_url.join(&tag_in_reg)?.as_str().to_string();
 
         // make deployment
+        println!("wooo");
         let deployment_name = function_to_deployment_name(function_name);
         let service_name = function_to_service_name(function_name);
+        println!("got service_name");
         let app_name = function_name.to_string();
-        let deployment = serde_json::from_value(serde_json::json!({
+        println!("... app_name is fine...");
+        let deployment_json = serde_json::json!({
             "apiVersion": "apps/v1",
             "kind": "Deployment",
             "metadata": {
@@ -121,20 +124,25 @@ impl DistributionPlatform for K8s {
                         "containers": [
                             {
                                 "name": tag_in_reg,
-                                "image": image_url.as_str(),
-                                "ports": {
-                                    "containerPort": 5000
-                                }
+                                "image": image_url,
+                                "ports": [
+                                    {
+                                        "containerPort": 5000
+                                    }
+                                ]
                             },
                         ]
                     }
                 }
-
             }
-        }))?;
+        });
+        println!("deployment_json generated, {:?}", deployment_json);
+        let deployment = serde_json::from_value(deployment_json)?;
+        println!("made deployment: {:?}", deployment);
         deployments
             .create(&PostParams::default(), &deployment)
             .await?;
+        println!("created deployment");
 
         // make service pointing to deployment
         let service = serde_json::from_value(serde_json::json!({
@@ -154,7 +162,9 @@ impl DistributionPlatform for K8s {
                 }
             }
         }))?;
+        println!("made service");
         let service = services.create(&PostParams::default(), &service).await?;
+        println!("created service");
         let service_ip = format!(
             "http://{}:5000",
             service
@@ -308,25 +318,41 @@ CMD RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run --release 127.0.0.1:500
             return Err(anyhow::anyhow!("docker image build failure"));
         }
 
-        // push image to local repo
         let image_tag = format!(
-            "{}:{}/{}",
-            registry_url.host_str().unwrap(),
+            "localhost:{}/{}",
             registry_url.port().unwrap(),
             function_name
         );
+
+        println!("image tag: {}", image_tag);
+
+        // tag image
+        let tag_args = format!("image tag {} {}", function_name, image_tag);
+        let tag_result = Command::new("docker")
+            .args(tag_args.as_str().split(' '))
+            .status()?;
+        if !tag_result.success() {
+            return Err(anyhow::anyhow!("docker image tag failure"));
+        }
+
+        println!("image tag worked: {}", tag_args);
+
+        // push image to local repo
         let push_status = Command::new("docker")
             .arg("push")
-            .arg(&image_tag)
+            .arg(image_tag.clone())
             .status()?;
+        println!("docker push command did not explode");
         if !push_status.success() {
             return Err(anyhow::anyhow!("docker image push failure"));
         }
 
         Ok(image_tag)
     })();
+    println!("removing build dir");
     // always remove the build directory, even on build error
     std::fs::remove_dir_all(build_dir_canonical)?;
+    println!("returning res");
     result
 }
 

From a7c645e36a4ad52bf4704a3d64fdab5277a28a8a Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Wed, 21 Oct 2020 09:09:11 +0200
Subject: [PATCH 039/145] use tokio runtime in examples because async-std seems
 not to work

---
 examples/kubernetes_example/Cargo.toml   | 4 ++--
 examples/kubernetes_example/src/main.rs  | 6 ++++--
 examples/local_queue_example/Cargo.toml  | 4 ++--
 examples/local_queue_example/src/main.rs | 7 ++++---
 4 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/examples/kubernetes_example/Cargo.toml b/examples/kubernetes_example/Cargo.toml
index d6e1afbb..2f1a178f 100644
--- a/examples/kubernetes_example/Cargo.toml
+++ b/examples/kubernetes_example/Cargo.toml
@@ -9,8 +9,8 @@ edition = "2018"
 
 [dependencies]
 rand = "0.7"
-async-std = "1"
+tokio = {version="0.3", features=["full"]}
 lazy_static = "1"
 futures = "0.3"
 cute = "0.3"
-turbolift = { path="../../" }
\ No newline at end of file
+turbolift = { path="../../" }
diff --git a/examples/kubernetes_example/src/main.rs b/examples/kubernetes_example/src/main.rs
index 98409ac8..891983d8 100644
--- a/examples/kubernetes_example/src/main.rs
+++ b/examples/kubernetes_example/src/main.rs
@@ -26,7 +26,8 @@ fn random_numbers() -> Vec<u64> {
 fn main() {
     let input = random_numbers();
     let futures = c![square(*int), for int in &input];
-    let output = async_std::task::block_on(try_join_all(futures)).unwrap();
+    let mut rt = tokio::runtime::Runtime::new().unwrap();
+    let output = rt.block_on(try_join_all(futures)).unwrap();
     println!("input: {:?}\noutput: {:?}", input, output);
 }
 
@@ -38,7 +39,8 @@ mod tests {
     fn it_works() {
         let input = random_numbers();
         let futures = c![square(*int), for int in &input];
-        let output = async_std::task::block_on(try_join_all(futures)).unwrap();
+        let mut rt = tokio::runtime::Runtime::new().unwrap();
+        let output = rt.block_on(try_join_all(futures)).unwrap();
         assert_eq!(
             output,
             input.into_iter().map(|x| x * x).collect::<Vec<u64>>()
diff --git a/examples/local_queue_example/Cargo.toml b/examples/local_queue_example/Cargo.toml
index 45530e03..60935de1 100644
--- a/examples/local_queue_example/Cargo.toml
+++ b/examples/local_queue_example/Cargo.toml
@@ -12,8 +12,8 @@ edition = "2018"
 rand = "0.7"
 futures = "0.3"
 lazy_static = "1"
-async-std = "1"
+tokio = {version="0.3", features=["full"]}
 turbolift = { path="../../" }
 
 # these are dependencies that need to be bundled into turbolift, but debugging how to reexport proc macros is hard.
-cached = "0.19"
\ No newline at end of file
+cached = "0.19"
diff --git a/examples/local_queue_example/src/main.rs b/examples/local_queue_example/src/main.rs
index 314c0418..31da3a03 100644
--- a/examples/local_queue_example/src/main.rs
+++ b/examples/local_queue_example/src/main.rs
@@ -1,7 +1,6 @@
 use std::sync::Mutex;
 
 extern crate proc_macro;
-use async_std::task;
 use futures::future::try_join_all;
 use rand;
 use turbolift::local_queue::LocalQueue;
@@ -27,7 +26,8 @@ fn main() {
         }
         v
     };
-    let output = task::block_on(try_join_all(futures)).unwrap();
+    let mut rt = tokio::runtime::Runtime::new().unwrap();
+    let output = rt.block_on(try_join_all(futures)).unwrap();
     println!("input: {:?}\noutput: {:?}", input, output);
 }
 
@@ -45,7 +45,8 @@ mod tests {
             }
             v
         };
-        let output = task::block_on(try_join_all(futures)).unwrap();
+        let mut rt = tokio::runtime::Runtime::new().unwrap();
+        let output = rt.block_on(try_join_all(futures)).unwrap();
         assert_eq!(input, output);
     }
 }

From f631ef58b04d45f3f0718b4fc5bc9172176154a0 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Wed, 21 Oct 2020 09:24:00 +0200
Subject: [PATCH 040/145] make examples exit with error codes if the output is
 not expected

---
 examples/kubernetes_example/src/main.rs  | 3 +++
 examples/local_queue_example/src/main.rs | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/examples/kubernetes_example/src/main.rs b/examples/kubernetes_example/src/main.rs
index 891983d8..0899deec 100644
--- a/examples/kubernetes_example/src/main.rs
+++ b/examples/kubernetes_example/src/main.rs
@@ -29,6 +29,9 @@ fn main() {
     let mut rt = tokio::runtime::Runtime::new().unwrap();
     let output = rt.block_on(try_join_all(futures)).unwrap();
     println!("input: {:?}\noutput: {:?}", input, output);
+    if output != c![x*x, for x in input] {
+        std::process::exit(1)
+    }
 }
 
 #[cfg(test)]
diff --git a/examples/local_queue_example/src/main.rs b/examples/local_queue_example/src/main.rs
index 31da3a03..0de00357 100644
--- a/examples/local_queue_example/src/main.rs
+++ b/examples/local_queue_example/src/main.rs
@@ -29,6 +29,9 @@ fn main() {
     let mut rt = tokio::runtime::Runtime::new().unwrap();
     let output = rt.block_on(try_join_all(futures)).unwrap();
     println!("input: {:?}\noutput: {:?}", input, output);
+    if output != input {
+        std::process::exit(1)
+    }
 }
 
 #[cfg(test)]

From bab40efea74121f2c13f45ae70ae615787d7549a Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Wed, 21 Oct 2020 10:10:50 +0200
Subject: [PATCH 041/145] switch from surf to reqwest, and set up a reusable
 client for request in the k8s struct

---
 turbolift_internals/Cargo.toml         |  4 ++--
 turbolift_internals/src/kubernetes.rs  | 23 ++++++++++++++++-------
 turbolift_internals/src/local_queue.rs | 16 ++++++++++------
 3 files changed, 28 insertions(+), 15 deletions(-)

diff --git a/turbolift_internals/Cargo.toml b/turbolift_internals/Cargo.toml
index a3c4c75e..bded6edd 100644
--- a/turbolift_internals/Cargo.toml
+++ b/turbolift_internals/Cargo.toml
@@ -19,6 +19,7 @@ tar = "0.4"
 toml = "0.5"
 cargo-toml2 = "1"
 tempfile = "3.1"
+reqwest = "0.10"
 tokio = { version = "0.2",    features = ["full"] }
 surf = "1.0.3"
 cute = "0.3"
@@ -27,7 +28,6 @@ url = "2"
 lazy_static = "1"
 anyhow = "1"
 cached = "0.19"
-async-std = "1.6"
 async-trait = "0.1"
 get_if_addrs = "0.5.3"
 regex = "1"
@@ -35,4 +35,4 @@ regex = "1"
 # kubernetes-specific requirements
 kube = "0.42.0"
 kube-runtime = "0.42.0"
-k8s-openapi = { version = "0.9.0", default-features = false, features = ["v1_17"] }
\ No newline at end of file
+k8s-openapi = { version = "0.9.0", default-features = false, features = ["v1_17"] }
diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 3a212896..fb768a3d 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -27,6 +27,7 @@ type ImageTag = String;
 pub struct K8s {
     max_scale_n: u32,
     fn_names_to_services: HashMap<String, Url>,
+    request_client: reqwest::Client,
 }
 
 impl K8s {
@@ -46,8 +47,19 @@ impl K8s {
         K8s {
             max_scale_n: max,
             fn_names_to_services: HashMap::new(),
+            request_client: reqwest::Client::new(),
         }
     }
+
+    async fn get(&self, query_url: Url) -> DistributionResult<String> {
+        Ok(self
+            .request_client
+            .get(query_url)
+            .send()
+            .await?
+            .text()
+            .await?)
+    }
 }
 
 impl Default for K8s {
@@ -202,16 +214,11 @@ impl DistributionPlatform for K8s {
         function_name: &str,
         params: ArgsString,
     ) -> DistributionResult<JsonResponse> {
-        async fn get(query_url: Url) -> String {
-            surf::get(query_url).recv_string().await.unwrap()
-        }
-
         // request from server
         let service_base_url = self.fn_names_to_services.get(function_name).unwrap();
         let args = "./".to_string() + function_name + "/" + &params;
         let query_url = service_base_url.join(&args)?;
-        let response = async_std::task::block_on(get(query_url));
-        // ^ todo not sure why futures are hanging here unless I wrap them in a new block_on?
+        let response = self.get(query_url).await?;
         Ok(response)
     }
 
@@ -359,7 +366,9 @@ CMD RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run --release 127.0.0.1:500
 impl Drop for K8s {
     fn drop(&mut self) {
         // delete the associated services and deployments from the functions we distributed
-        async_std::task::block_on(async {
+
+        let mut rt = tokio::runtime::Runtime::new().unwrap();
+        rt.block_on(async {
             let deployment_client = Client::try_default().await.unwrap();
             let deployments: Api<Deployment> = Api::namespaced(deployment_client, K8S_NAMESPACE);
             let service_client = Client::try_default().await.unwrap();
diff --git a/turbolift_internals/src/local_queue.rs b/turbolift_internals/src/local_queue.rs
index 00130db3..9f308b28 100644
--- a/turbolift_internals/src/local_queue.rs
+++ b/turbolift_internals/src/local_queue.rs
@@ -30,6 +30,15 @@ impl LocalQueue {
     pub fn new() -> LocalQueue {
         Default::default()
     }
+
+    async fn get(query_url: Url) -> String {
+        reqwest::get(query_url)
+            .await
+            .expect("error unwrapping local queue get response")
+            .text()
+            .await
+            .expect("error unwrapping local queue text response")
+    }
 }
 
 #[async_trait]
@@ -57,10 +66,6 @@ impl DistributionPlatform for LocalQueue {
         function_name: &str,
         params: ArgsString,
     ) -> DistributionResult<JsonResponse> {
-        async fn get(query_url: Url) -> String {
-            surf::get(query_url).recv_string().await.unwrap()
-        }
-
         let address_and_port = {
             if self.fn_name_to_address.contains_key(function_name) {
                 // the server is already initialized.
@@ -91,8 +96,7 @@ impl DistributionPlatform for LocalQueue {
         // request from server
         let prefixed_params = "./".to_string() + function_name + "/" + &params;
         let query_url = address_and_port.join(&prefixed_params)?;
-        let response = async_std::task::block_on(get(query_url));
-        // ^ todo not sure why futures are hanging here unless I wrap them in a new block_on?
+        let response = Self::get(query_url).await;
         Ok(response)
     }
 

From 8d25e0df00db7364174d579e290eadc1c02b1693 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Wed, 21 Oct 2020 10:43:52 +0200
Subject: [PATCH 042/145] do not export async-std, we use tokio now

---
 Cargo.toml                  | 5 ++---
 src/lib.rs                  | 2 --
 turbolift_macros/src/lib.rs | 1 -
 3 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index d4e1b3f9..fc97981b 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -8,13 +8,12 @@ readme = "README.md"
 license = "Hippocratic-2.1"
 
 [features]
-distributed = ["chrono", "async-std", "turbolift_macros/distributed"]
+distributed = ["chrono", "turbolift_macros/distributed"]
 # todo we can optimize reqs for children with this load
 
 [dependencies]
 turbolift_macros = { path = "./turbolift_macros" }
 turbolift_internals = { path = "./turbolift_internals" }
-async-std = { version = "1.6", optional = true }
 chrono = { version = "0.4", optional = true }
 actix-web = { version = "3" }
-serde_json = { version = "1" }
\ No newline at end of file
+serde_json = { version = "1" }
diff --git a/src/lib.rs b/src/lib.rs
index 4ee38be5..ad75fc97 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,6 +1,4 @@
 #[cfg(feature = "distributed")]
-pub use async_std;
-#[cfg(feature = "distributed")]
 pub use chrono;
 
 pub use actix_web;
diff --git a/turbolift_macros/src/lib.rs b/turbolift_macros/src/lib.rs
index cbb9828b..321b7b27 100644
--- a/turbolift_macros/src/lib.rs
+++ b/turbolift_macros/src/lib.rs
@@ -160,7 +160,6 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
             use std::time::Duration;
             use turbolift::distributed_platform::DistributionPlatform;
             use turbolift::DistributionResult;
-            use turbolift::async_std::task;
 
             let mut platform = #distribution_platform.lock()?;
 

From 1a423f318b20fa8ea05488fdb08f45fad84ee01e Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Wed, 21 Oct 2020 10:58:25 +0200
Subject: [PATCH 043/145] temp bugfix: for some reason, async traits don't seem
 to handle futures in impls correctly. the error seems to be because the
 runtime environment is not accessible/out of scope

---
 turbolift_internals/src/kubernetes.rs  | 5 ++++-
 turbolift_internals/src/local_queue.rs | 5 ++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index fb768a3d..28afe9d9 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -218,7 +218,10 @@ impl DistributionPlatform for K8s {
         let service_base_url = self.fn_names_to_services.get(function_name).unwrap();
         let args = "./".to_string() + function_name + "/" + &params;
         let query_url = service_base_url.join(&args)?;
-        let response = self.get(query_url).await?;
+
+        let mut rt = tokio::runtime::Runtime::new().unwrap();
+        let response = rt.block_on(self.get(query_url))?;
+        // ^ todo find out why this code errors if we don't spawn another runtime here
         Ok(response)
     }
 
diff --git a/turbolift_internals/src/local_queue.rs b/turbolift_internals/src/local_queue.rs
index 9f308b28..c50b71f1 100644
--- a/turbolift_internals/src/local_queue.rs
+++ b/turbolift_internals/src/local_queue.rs
@@ -96,7 +96,10 @@ impl DistributionPlatform for LocalQueue {
         // request from server
         let prefixed_params = "./".to_string() + function_name + "/" + &params;
         let query_url = address_and_port.join(&prefixed_params)?;
-        let response = Self::get(query_url).await;
+
+        let mut rt = tokio::runtime::Runtime::new().unwrap();
+        let response = rt.block_on(Self::get(query_url));
+        // ^ todo find out why this code errors if we don't spawn another runtime here
         Ok(response)
     }
 

From 38237d69c4fea5907939f6eff0e67be466b99084 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Wed, 21 Oct 2020 11:50:11 +0200
Subject: [PATCH 044/145] always use tokio 0.2 instead of 0.3

---
 examples/kubernetes_example/Cargo.toml   |  2 +-
 examples/kubernetes_example/src/main.rs  | 12 ++++++++++--
 examples/local_queue_example/Cargo.toml  |  2 +-
 examples/local_queue_example/src/main.rs | 12 ++++++++++--
 turbolift_internals/Cargo.toml           |  2 +-
 turbolift_internals/src/kubernetes.rs    | 11 ++++++-----
 turbolift_internals/src/local_queue.rs   |  4 +---
 7 files changed, 30 insertions(+), 15 deletions(-)

diff --git a/examples/kubernetes_example/Cargo.toml b/examples/kubernetes_example/Cargo.toml
index 2f1a178f..c8fde44a 100644
--- a/examples/kubernetes_example/Cargo.toml
+++ b/examples/kubernetes_example/Cargo.toml
@@ -9,7 +9,7 @@ edition = "2018"
 
 [dependencies]
 rand = "0.7"
-tokio = {version="0.3", features=["full"]}
+tokio = {version="0.2.22", features=["full"]}
 lazy_static = "1"
 futures = "0.3"
 cute = "0.3"
diff --git a/examples/kubernetes_example/src/main.rs b/examples/kubernetes_example/src/main.rs
index 0899deec..6a6d57f7 100644
--- a/examples/kubernetes_example/src/main.rs
+++ b/examples/kubernetes_example/src/main.rs
@@ -26,7 +26,11 @@ fn random_numbers() -> Vec<u64> {
 fn main() {
     let input = random_numbers();
     let futures = c![square(*int), for int in &input];
-    let mut rt = tokio::runtime::Runtime::new().unwrap();
+    let mut rt = tokio::runtime::Builder::new()
+        .threaded_scheduler()
+        .enable_all()
+        .build()
+        .expect("error starting runtime");
     let output = rt.block_on(try_join_all(futures)).unwrap();
     println!("input: {:?}\noutput: {:?}", input, output);
     if output != c![x*x, for x in input] {
@@ -42,7 +46,11 @@ mod tests {
     fn it_works() {
         let input = random_numbers();
         let futures = c![square(*int), for int in &input];
-        let mut rt = tokio::runtime::Runtime::new().unwrap();
+        let mut rt = tokio::runtime::Builder::new()
+            .threaded_scheduler()
+            .enable_all()
+            .build()
+            .expect("error starting runtime");
         let output = rt.block_on(try_join_all(futures)).unwrap();
         assert_eq!(
             output,
diff --git a/examples/local_queue_example/Cargo.toml b/examples/local_queue_example/Cargo.toml
index 60935de1..87bba44c 100644
--- a/examples/local_queue_example/Cargo.toml
+++ b/examples/local_queue_example/Cargo.toml
@@ -12,7 +12,7 @@ edition = "2018"
 rand = "0.7"
 futures = "0.3"
 lazy_static = "1"
-tokio = {version="0.3", features=["full"]}
+tokio = {version="0.2.22", features=["full"]}
 turbolift = { path="../../" }
 
 # these are dependencies that need to be bundled into turbolift, but debugging how to reexport proc macros is hard.
diff --git a/examples/local_queue_example/src/main.rs b/examples/local_queue_example/src/main.rs
index 0de00357..12ff9741 100644
--- a/examples/local_queue_example/src/main.rs
+++ b/examples/local_queue_example/src/main.rs
@@ -26,7 +26,11 @@ fn main() {
         }
         v
     };
-    let mut rt = tokio::runtime::Runtime::new().unwrap();
+    let mut rt = tokio::runtime::Builder::new()
+        .threaded_scheduler()
+        .enable_all()
+        .build()
+        .expect("error starting runtime");
     let output = rt.block_on(try_join_all(futures)).unwrap();
     println!("input: {:?}\noutput: {:?}", input, output);
     if output != input {
@@ -48,7 +52,11 @@ mod tests {
             }
             v
         };
-        let mut rt = tokio::runtime::Runtime::new().unwrap();
+        let mut rt = tokio::runtime::Builder::new()
+            .threaded_scheduler()
+            .enable_all()
+            .build()
+            .expect("error starting runtime");
         let output = rt.block_on(try_join_all(futures)).unwrap();
         assert_eq!(input, output);
     }
diff --git a/turbolift_internals/Cargo.toml b/turbolift_internals/Cargo.toml
index bded6edd..6bbbd162 100644
--- a/turbolift_internals/Cargo.toml
+++ b/turbolift_internals/Cargo.toml
@@ -20,7 +20,7 @@ toml = "0.5"
 cargo-toml2 = "1"
 tempfile = "3.1"
 reqwest = "0.10"
-tokio = { version = "0.2",    features = ["full"] }
+tokio = { version = "0.2.22",    features = ["full"] }
 surf = "1.0.3"
 cute = "0.3"
 rand = "0.7"
diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 28afe9d9..6269c90d 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -219,9 +219,7 @@ impl DistributionPlatform for K8s {
         let args = "./".to_string() + function_name + "/" + &params;
         let query_url = service_base_url.join(&args)?;
 
-        let mut rt = tokio::runtime::Runtime::new().unwrap();
-        let response = rt.block_on(self.get(query_url))?;
-        // ^ todo find out why this code errors if we don't spawn another runtime here
+        let response = self.get(query_url).await?;
         Ok(response)
     }
 
@@ -369,8 +367,11 @@ CMD RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run --release 127.0.0.1:500
 impl Drop for K8s {
     fn drop(&mut self) {
         // delete the associated services and deployments from the functions we distributed
-
-        let mut rt = tokio::runtime::Runtime::new().unwrap();
+        let mut rt = tokio::runtime::Builder::new()
+            .basic_scheduler()
+            .enable_all()
+            .build()
+            .expect("unable to instantiate basic scheduler in K8s Drop impl");
         rt.block_on(async {
             let deployment_client = Client::try_default().await.unwrap();
             let deployments: Api<Deployment> = Api::namespaced(deployment_client, K8S_NAMESPACE);
diff --git a/turbolift_internals/src/local_queue.rs b/turbolift_internals/src/local_queue.rs
index c50b71f1..f407c7d2 100644
--- a/turbolift_internals/src/local_queue.rs
+++ b/turbolift_internals/src/local_queue.rs
@@ -97,9 +97,7 @@ impl DistributionPlatform for LocalQueue {
         let prefixed_params = "./".to_string() + function_name + "/" + &params;
         let query_url = address_and_port.join(&prefixed_params)?;
 
-        let mut rt = tokio::runtime::Runtime::new().unwrap();
-        let response = rt.block_on(Self::get(query_url));
-        // ^ todo find out why this code errors if we don't spawn another runtime here
+        let response = Self::get(query_url).await;
         Ok(response)
     }
 

From 7db9b2a0bbe9c3326fd3dfdd0bd0663ff0691fae Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Thu, 22 Oct 2020 15:42:21 +0200
Subject: [PATCH 045/145] try using a handle to avoid the hanging issue from
 https://github.com/DominicBurkart/turbolift/pull/2#issuecomment-713584765

---
 turbolift_internals/src/kubernetes.rs  | 4 ++--
 turbolift_internals/src/local_queue.rs | 3 ++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 6269c90d..fff83133 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -218,8 +218,8 @@ impl DistributionPlatform for K8s {
         let service_base_url = self.fn_names_to_services.get(function_name).unwrap();
         let args = "./".to_string() + function_name + "/" + &params;
         let query_url = service_base_url.join(&args)?;
-
-        let response = self.get(query_url).await?;
+        let handle = tokio::runtime::Handle::current();
+        let response = handle.block_on(self.get(query_url))?;
         Ok(response)
     }
 
diff --git a/turbolift_internals/src/local_queue.rs b/turbolift_internals/src/local_queue.rs
index f407c7d2..18f74e73 100644
--- a/turbolift_internals/src/local_queue.rs
+++ b/turbolift_internals/src/local_queue.rs
@@ -97,7 +97,8 @@ impl DistributionPlatform for LocalQueue {
         let prefixed_params = "./".to_string() + function_name + "/" + &params;
         let query_url = address_and_port.join(&prefixed_params)?;
 
-        let response = Self::get(query_url).await;
+        let handle = tokio::runtime::Handle::current();
+        let response = handle.block_on(Self::get(query_url));
         Ok(response)
     }
 

From 8746ed9abfd428b4f683d09de79082ca1e8106a9 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Thu, 22 Oct 2020 20:31:15 +0200
Subject: [PATCH 046/145] remove attempts at fixing hanging issue & inline
 dispatch GET request - re-upgrade tokio to 0.3 - remove attempts to add
 additional block within tokio environment to avoid hang

---
 examples/kubernetes_example/Cargo.toml   |  2 +-
 examples/kubernetes_example/src/main.rs  | 10 ++++-----
 examples/local_queue_example/Cargo.toml  |  2 +-
 examples/local_queue_example/src/main.rs | 10 ++++-----
 turbolift_internals/Cargo.toml           |  2 +-
 turbolift_internals/src/kubernetes.rs    | 26 ++++++++----------------
 turbolift_internals/src/local_queue.rs   | 20 ++++++++----------
 7 files changed, 27 insertions(+), 45 deletions(-)

diff --git a/examples/kubernetes_example/Cargo.toml b/examples/kubernetes_example/Cargo.toml
index c8fde44a..2f1a178f 100644
--- a/examples/kubernetes_example/Cargo.toml
+++ b/examples/kubernetes_example/Cargo.toml
@@ -9,7 +9,7 @@ edition = "2018"
 
 [dependencies]
 rand = "0.7"
-tokio = {version="0.2.22", features=["full"]}
+tokio = {version="0.3", features=["full"]}
 lazy_static = "1"
 futures = "0.3"
 cute = "0.3"
diff --git a/examples/kubernetes_example/src/main.rs b/examples/kubernetes_example/src/main.rs
index 6a6d57f7..9b6d85a3 100644
--- a/examples/kubernetes_example/src/main.rs
+++ b/examples/kubernetes_example/src/main.rs
@@ -26,11 +26,10 @@ fn random_numbers() -> Vec<u64> {
 fn main() {
     let input = random_numbers();
     let futures = c![square(*int), for int in &input];
-    let mut rt = tokio::runtime::Builder::new()
-        .threaded_scheduler()
+    let mut rt = tokio::runtime::Builder::new_multi_thread()
         .enable_all()
         .build()
-        .expect("error starting runtime");
+        .unwrap();
     let output = rt.block_on(try_join_all(futures)).unwrap();
     println!("input: {:?}\noutput: {:?}", input, output);
     if output != c![x*x, for x in input] {
@@ -46,11 +45,10 @@ mod tests {
     fn it_works() {
         let input = random_numbers();
         let futures = c![square(*int), for int in &input];
-        let mut rt = tokio::runtime::Builder::new()
-            .threaded_scheduler()
+        let mut rt = tokio::runtime::Builder::new_multi_thread()
             .enable_all()
             .build()
-            .expect("error starting runtime");
+            .unwrap();
         let output = rt.block_on(try_join_all(futures)).unwrap();
         assert_eq!(
             output,
diff --git a/examples/local_queue_example/Cargo.toml b/examples/local_queue_example/Cargo.toml
index 87bba44c..60935de1 100644
--- a/examples/local_queue_example/Cargo.toml
+++ b/examples/local_queue_example/Cargo.toml
@@ -12,7 +12,7 @@ edition = "2018"
 rand = "0.7"
 futures = "0.3"
 lazy_static = "1"
-tokio = {version="0.2.22", features=["full"]}
+tokio = {version="0.3", features=["full"]}
 turbolift = { path="../../" }
 
 # these are dependencies that need to be bundled into turbolift, but debugging how to reexport proc macros is hard.
diff --git a/examples/local_queue_example/src/main.rs b/examples/local_queue_example/src/main.rs
index 12ff9741..c79f0d12 100644
--- a/examples/local_queue_example/src/main.rs
+++ b/examples/local_queue_example/src/main.rs
@@ -26,11 +26,10 @@ fn main() {
         }
         v
     };
-    let mut rt = tokio::runtime::Builder::new()
-        .threaded_scheduler()
+    let mut rt = tokio::runtime::Builder::new_multi_thread()
         .enable_all()
         .build()
-        .expect("error starting runtime");
+        .unwrap();
     let output = rt.block_on(try_join_all(futures)).unwrap();
     println!("input: {:?}\noutput: {:?}", input, output);
     if output != input {
@@ -52,11 +51,10 @@ mod tests {
             }
             v
         };
-        let mut rt = tokio::runtime::Builder::new()
-            .threaded_scheduler()
+        let mut rt = tokio::runtime::Builder::new_multi_thread()
             .enable_all()
             .build()
-            .expect("error starting runtime");
+            .unwrap();
         let output = rt.block_on(try_join_all(futures)).unwrap();
         assert_eq!(input, output);
     }
diff --git a/turbolift_internals/Cargo.toml b/turbolift_internals/Cargo.toml
index 6bbbd162..aaefcab5 100644
--- a/turbolift_internals/Cargo.toml
+++ b/turbolift_internals/Cargo.toml
@@ -20,7 +20,7 @@ toml = "0.5"
 cargo-toml2 = "1"
 tempfile = "3.1"
 reqwest = "0.10"
-tokio = { version = "0.2.22",    features = ["full"] }
+tokio = { version = "0.3",    features = ["full"] }
 surf = "1.0.3"
 cute = "0.3"
 rand = "0.7"
diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index fff83133..54df6be4 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -50,16 +50,6 @@ impl K8s {
             request_client: reqwest::Client::new(),
         }
     }
-
-    async fn get(&self, query_url: Url) -> DistributionResult<String> {
-        Ok(self
-            .request_client
-            .get(query_url)
-            .send()
-            .await?
-            .text()
-            .await?)
-    }
 }
 
 impl Default for K8s {
@@ -218,9 +208,13 @@ impl DistributionPlatform for K8s {
         let service_base_url = self.fn_names_to_services.get(function_name).unwrap();
         let args = "./".to_string() + function_name + "/" + &params;
         let query_url = service_base_url.join(&args)?;
-        let handle = tokio::runtime::Handle::current();
-        let response = handle.block_on(self.get(query_url))?;
-        Ok(response)
+        Ok(self
+            .request_client
+            .get(query_url)
+            .send()
+            .await?
+            .text()
+            .await?)
     }
 
     fn has_declared(&self, fn_name: &str) -> bool {
@@ -367,11 +361,7 @@ CMD RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run --release 127.0.0.1:500
 impl Drop for K8s {
     fn drop(&mut self) {
         // delete the associated services and deployments from the functions we distributed
-        let mut rt = tokio::runtime::Builder::new()
-            .basic_scheduler()
-            .enable_all()
-            .build()
-            .expect("unable to instantiate basic scheduler in K8s Drop impl");
+        let rt = tokio::runtime::Runtime::new().unwrap();
         rt.block_on(async {
             let deployment_client = Client::try_default().await.unwrap();
             let deployments: Api<Deployment> = Api::namespaced(deployment_client, K8S_NAMESPACE);
diff --git a/turbolift_internals/src/local_queue.rs b/turbolift_internals/src/local_queue.rs
index 18f74e73..31a6a699 100644
--- a/turbolift_internals/src/local_queue.rs
+++ b/turbolift_internals/src/local_queue.rs
@@ -24,21 +24,13 @@ pub struct LocalQueue {
     fn_name_to_address: HashMap<FunctionName, AddressAndPort>, // todo hardcoded rn
     fn_name_to_process: HashMap<FunctionName, Child>,
     fn_name_to_binary_path: HashMap<FunctionName, std::path::PathBuf>,
+    request_client: reqwest::Client,
 }
 
 impl LocalQueue {
     pub fn new() -> LocalQueue {
         Default::default()
     }
-
-    async fn get(query_url: Url) -> String {
-        reqwest::get(query_url)
-            .await
-            .expect("error unwrapping local queue get response")
-            .text()
-            .await
-            .expect("error unwrapping local queue text response")
-    }
 }
 
 #[async_trait]
@@ -97,9 +89,13 @@ impl DistributionPlatform for LocalQueue {
         let prefixed_params = "./".to_string() + function_name + "/" + &params;
         let query_url = address_and_port.join(&prefixed_params)?;
 
-        let handle = tokio::runtime::Handle::current();
-        let response = handle.block_on(Self::get(query_url));
-        Ok(response)
+        Ok(self
+            .request_client
+            .get(query_url)
+            .send()
+            .await?
+            .text()
+            .await?)
     }
 
     fn has_declared(&self, fn_name: &str) -> bool {

From ee66f63ccfb669e0e031f5f4ecde0765e1bbdea5 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Fri, 23 Oct 2020 10:15:29 +0200
Subject: [PATCH 047/145] use patched kube dependency

---
 turbolift_internals/Cargo.toml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/turbolift_internals/Cargo.toml b/turbolift_internals/Cargo.toml
index aaefcab5..33d911b6 100644
--- a/turbolift_internals/Cargo.toml
+++ b/turbolift_internals/Cargo.toml
@@ -33,6 +33,6 @@ get_if_addrs = "0.5.3"
 regex = "1"
 
 # kubernetes-specific requirements
-kube = "0.42.0"
-kube-runtime = "0.42.0"
+kube = { repository = "https://github.com/DominicBurkart/kube-rs" }
+kube-runtime = { repository = "https://github.com/DominicBurkart/kube-rs" }
 k8s-openapi = { version = "0.9.0", default-features = false, features = ["v1_17"] }

From 3eb4a4ef54ef52ceec3e72f3c1dfc687c2b33c43 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Fri, 23 Oct 2020 11:51:12 +0200
Subject: [PATCH 048/145] print before and after presumed error site

---
 turbolift_internals/src/kubernetes.rs  | 7 +++++--
 turbolift_internals/src/local_queue.rs | 7 +++++--
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 54df6be4..83b092bb 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -208,13 +208,16 @@ impl DistributionPlatform for K8s {
         let service_base_url = self.fn_names_to_services.get(function_name).unwrap();
         let args = "./".to_string() + function_name + "/" + &params;
         let query_url = service_base_url.join(&args)?;
-        Ok(self
+        println!("sending dispatch request to {:?}", query_url);
+        let resp = Ok(self
             .request_client
             .get(query_url)
             .send()
             .await?
             .text()
-            .await?)
+            .await?);
+        println!("dispatch returning: {:?}", resp);
+        resp
     }
 
     fn has_declared(&self, fn_name: &str) -> bool {
diff --git a/turbolift_internals/src/local_queue.rs b/turbolift_internals/src/local_queue.rs
index 31a6a699..adf10100 100644
--- a/turbolift_internals/src/local_queue.rs
+++ b/turbolift_internals/src/local_queue.rs
@@ -89,13 +89,16 @@ impl DistributionPlatform for LocalQueue {
         let prefixed_params = "./".to_string() + function_name + "/" + &params;
         let query_url = address_and_port.join(&prefixed_params)?;
 
-        Ok(self
+        println!("sending dispatch request to {:?}", query_url);
+        let resp = Ok(self
             .request_client
             .get(query_url)
             .send()
             .await?
             .text()
-            .await?)
+            .await?);
+        println!("dispatch returning: {:?}", resp);
+        resp
     }
 
     fn has_declared(&self, fn_name: &str) -> bool {

From 9d06cdb53deeec346d5a7a85dd544724bb8e618a Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Fri, 23 Oct 2020 12:45:26 +0200
Subject: [PATCH 049/145] do not use patched kube, do use patched reqwest

---
 turbolift_internals/Cargo.toml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/turbolift_internals/Cargo.toml b/turbolift_internals/Cargo.toml
index 33d911b6..41063078 100644
--- a/turbolift_internals/Cargo.toml
+++ b/turbolift_internals/Cargo.toml
@@ -19,7 +19,7 @@ tar = "0.4"
 toml = "0.5"
 cargo-toml2 = "1"
 tempfile = "3.1"
-reqwest = "0.10"
+reqwest = {repository="https://github.com/DominicBurkart/reqwest"}
 tokio = { version = "0.3",    features = ["full"] }
 surf = "1.0.3"
 cute = "0.3"
@@ -33,6 +33,6 @@ get_if_addrs = "0.5.3"
 regex = "1"
 
 # kubernetes-specific requirements
-kube = { repository = "https://github.com/DominicBurkart/kube-rs" }
-kube-runtime = { repository = "https://github.com/DominicBurkart/kube-rs" }
+kube = "0.43.0"
+kube-runtime = "0.43.0"
 k8s-openapi = { version = "0.9.0", default-features = false, features = ["v1_17"] }

From 6296dad0292670ccec48d482ae1fffdfd05afc08 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Fri, 23 Oct 2020 12:45:51 +0200
Subject: [PATCH 050/145] remove unused import

---
 turbolift_internals/Cargo.toml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/turbolift_internals/Cargo.toml b/turbolift_internals/Cargo.toml
index 41063078..f8f0ab6f 100644
--- a/turbolift_internals/Cargo.toml
+++ b/turbolift_internals/Cargo.toml
@@ -21,7 +21,6 @@ cargo-toml2 = "1"
 tempfile = "3.1"
 reqwest = {repository="https://github.com/DominicBurkart/reqwest"}
 tokio = { version = "0.3",    features = ["full"] }
-surf = "1.0.3"
 cute = "0.3"
 rand = "0.7"
 url = "2"

From cbd8129c383f2f52bcfc31d9657d78e342386a16 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Fri, 23 Oct 2020 15:01:03 +0200
Subject: [PATCH 051/145] use tokio 0.2 instead of 0.3.1

---
 examples/kubernetes_example/Cargo.toml   | 2 +-
 examples/kubernetes_example/src/main.rs  | 6 ++++--
 examples/local_queue_example/Cargo.toml  | 2 +-
 examples/local_queue_example/src/main.rs | 6 ++++--
 turbolift_internals/Cargo.toml           | 2 +-
 turbolift_internals/src/kubernetes.rs    | 2 +-
 6 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/examples/kubernetes_example/Cargo.toml b/examples/kubernetes_example/Cargo.toml
index 2f1a178f..694a3fc8 100644
--- a/examples/kubernetes_example/Cargo.toml
+++ b/examples/kubernetes_example/Cargo.toml
@@ -9,7 +9,7 @@ edition = "2018"
 
 [dependencies]
 rand = "0.7"
-tokio = {version="0.3", features=["full"]}
+tokio = {version="0.2", features=["full"]}
 lazy_static = "1"
 futures = "0.3"
 cute = "0.3"
diff --git a/examples/kubernetes_example/src/main.rs b/examples/kubernetes_example/src/main.rs
index 9b6d85a3..100716d4 100644
--- a/examples/kubernetes_example/src/main.rs
+++ b/examples/kubernetes_example/src/main.rs
@@ -26,7 +26,8 @@ fn random_numbers() -> Vec<u64> {
 fn main() {
     let input = random_numbers();
     let futures = c![square(*int), for int in &input];
-    let mut rt = tokio::runtime::Builder::new_multi_thread()
+    let mut rt = tokio::runtime::Builder::new()
+        .threaded_scheduler()
         .enable_all()
         .build()
         .unwrap();
@@ -45,7 +46,8 @@ mod tests {
     fn it_works() {
         let input = random_numbers();
         let futures = c![square(*int), for int in &input];
-        let mut rt = tokio::runtime::Builder::new_multi_thread()
+        let mut rt = tokio::runtime::Builder::new()
+            .threaded_scheduler()
             .enable_all()
             .build()
             .unwrap();
diff --git a/examples/local_queue_example/Cargo.toml b/examples/local_queue_example/Cargo.toml
index 60935de1..79681087 100644
--- a/examples/local_queue_example/Cargo.toml
+++ b/examples/local_queue_example/Cargo.toml
@@ -12,7 +12,7 @@ edition = "2018"
 rand = "0.7"
 futures = "0.3"
 lazy_static = "1"
-tokio = {version="0.3", features=["full"]}
+tokio = {version="0.2", features=["full"]}
 turbolift = { path="../../" }
 
 # these are dependencies that need to be bundled into turbolift, but debugging how to reexport proc macros is hard.
diff --git a/examples/local_queue_example/src/main.rs b/examples/local_queue_example/src/main.rs
index c79f0d12..589df11e 100644
--- a/examples/local_queue_example/src/main.rs
+++ b/examples/local_queue_example/src/main.rs
@@ -26,7 +26,8 @@ fn main() {
         }
         v
     };
-    let mut rt = tokio::runtime::Builder::new_multi_thread()
+    let mut rt = tokio::runtime::Builder::new()
+        .threaded_scheduler()
         .enable_all()
         .build()
         .unwrap();
@@ -51,7 +52,8 @@ mod tests {
             }
             v
         };
-        let mut rt = tokio::runtime::Builder::new_multi_thread()
+        let mut rt = tokio::runtime::Builder::new()
+            .threaded_scheduler()
             .enable_all()
             .build()
             .unwrap();
diff --git a/turbolift_internals/Cargo.toml b/turbolift_internals/Cargo.toml
index f8f0ab6f..d86c81b0 100644
--- a/turbolift_internals/Cargo.toml
+++ b/turbolift_internals/Cargo.toml
@@ -20,7 +20,7 @@ toml = "0.5"
 cargo-toml2 = "1"
 tempfile = "3.1"
 reqwest = {repository="https://github.com/DominicBurkart/reqwest"}
-tokio = { version = "0.3",    features = ["full"] }
+tokio = { version = "0.2",    features = ["full"] }
 cute = "0.3"
 rand = "0.7"
 url = "2"
diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 83b092bb..49c3aafd 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -364,7 +364,7 @@ CMD RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run --release 127.0.0.1:500
 impl Drop for K8s {
     fn drop(&mut self) {
         // delete the associated services and deployments from the functions we distributed
-        let rt = tokio::runtime::Runtime::new().unwrap();
+        let mut rt = tokio::runtime::Runtime::new().unwrap();
         rt.block_on(async {
             let deployment_client = Client::try_default().await.unwrap();
             let deployments: Api<Deployment> = Api::namespaced(deployment_client, K8S_NAMESPACE);

From 8e42ec0f024ae3bc8f49e11059f289eda03a5e77 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sat, 24 Oct 2020 10:39:01 +0200
Subject: [PATCH 052/145] use tokio delay_for fn in local_queue_example after
 registering a new server, instead of sleeping thread

---
 turbolift_internals/Cargo.toml         | 2 +-
 turbolift_internals/src/local_queue.rs | 4 +---
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/turbolift_internals/Cargo.toml b/turbolift_internals/Cargo.toml
index d86c81b0..8f05cd6d 100644
--- a/turbolift_internals/Cargo.toml
+++ b/turbolift_internals/Cargo.toml
@@ -20,7 +20,7 @@ toml = "0.5"
 cargo-toml2 = "1"
 tempfile = "3.1"
 reqwest = {repository="https://github.com/DominicBurkart/reqwest"}
-tokio = { version = "0.2",    features = ["full"] }
+tokio = { version = "0.2", features = ["full"] }
 cute = "0.3"
 rand = "0.7"
 url = "2"
diff --git a/turbolift_internals/src/local_queue.rs b/turbolift_internals/src/local_queue.rs
index adf10100..33d6bf05 100644
--- a/turbolift_internals/src/local_queue.rs
+++ b/turbolift_internals/src/local_queue.rs
@@ -2,7 +2,6 @@ extern crate proc_macro;
 use std::collections::HashMap;
 use std::fs;
 use std::path::Path;
-use std::thread::sleep;
 use std::time::Duration;
 
 use async_trait::async_trait;
@@ -74,9 +73,8 @@ impl DistributionPlatform for LocalQueue {
                 let server_handle = Command::new(executable)
                     .arg(&server_address_and_port_str)
                     .spawn()?;
-                sleep(Duration::from_secs(30));
+                tokio::time::delay_for(Duration::from_secs(60)).await;
                 // ^ sleep to make sure the server is initialized before continuing
-                // todo: here and with the GET request, futures hang indefinitely. To investigate.
                 self.fn_name_to_address
                     .insert(function_name.to_string(), server_url.clone());
                 self.fn_name_to_process

From c3167d9a10b1f19715a14c38bf6b3fb8354b3f8f Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sat, 24 Oct 2020 11:49:30 +0200
Subject: [PATCH 053/145] add debugging comments

---
 turbolift_internals/src/local_queue.rs | 3 +++
 turbolift_macros/src/lib.rs            | 6 ++++++
 2 files changed, 9 insertions(+)

diff --git a/turbolift_internals/src/local_queue.rs b/turbolift_internals/src/local_queue.rs
index 33d6bf05..e84aeb4c 100644
--- a/turbolift_internals/src/local_queue.rs
+++ b/turbolift_internals/src/local_queue.rs
@@ -70,10 +70,13 @@ impl DistributionPlatform for LocalQueue {
                 let server_url: AddressAndPort =
                     Url::parse(&("http://".to_string() + server_address_and_port_str))?;
                 let executable = self.fn_name_to_binary_path.get(function_name).unwrap();
+                println!("spawning");
                 let server_handle = Command::new(executable)
                     .arg(&server_address_and_port_str)
                     .spawn()?;
+                println!("delaying");
                 tokio::time::delay_for(Duration::from_secs(60)).await;
+                println!("delay completed");
                 // ^ sleep to make sure the server is initialized before continuing
                 self.fn_name_to_address
                     .insert(function_name.to_string(), server_url.clone());
diff --git a/turbolift_macros/src/lib.rs b/turbolift_macros/src/lib.rs
index 321b7b27..ed85e02c 100644
--- a/turbolift_macros/src/lib.rs
+++ b/turbolift_macros/src/lib.rs
@@ -135,6 +135,7 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
     // ^ todo
 
     // compress project source files
+    println!("on: build complete");
     let project_source_binary = {
         let tar = extract_function::make_compressed_proj_src(&function_cache_proj_path);
         let tar_file = CACHE_PATH.join(original_target_function_name.clone() + "_source.tar");
@@ -149,6 +150,7 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
         ))
         .expect("syntax error while embedding project tar.")
     };
+    println!("on: project_source_binary complete");
 
     // generate API function for the microservice
     let declare_and_dispatch = q! {
@@ -164,18 +166,22 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
             let mut platform = #distribution_platform.lock()?;
 
             if !platform.has_declared(#original_target_function_name) {
+                println!("launching declare");
                 platform
                     .declare(#original_target_function_name, #project_source_binary)
                     .await?;
+                println!("declare completed");
             }
 
             let params = #params_vec.join("/");
 
+            println!("launching dispatch");
             let resp_string = platform
                 .dispatch(
                     #original_target_function_name,
                     params.to_string()
                 ).await?;
+            println!("dispatch completed");
             Ok(turbolift::serde_json::from_str(&resp_string)?)
         }
     };

From bc493c4a723f9be15e94770929c150bca3b8e461 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Mon, 26 Oct 2020 11:15:24 +0100
Subject: [PATCH 054/145] switch from tokio 0.2 to 0.3 with compat layer for
 0.2 deps (kube and reqwest)

---
 Cargo.toml                               |  1 +
 examples/kubernetes_example/Cargo.toml   |  2 +-
 examples/kubernetes_example/src/main.rs  | 12 ++----------
 examples/local_queue_example/Cargo.toml  |  2 +-
 examples/local_queue_example/src/main.rs | 13 +++----------
 src/lib.rs                               |  1 +
 turbolift_internals/Cargo.toml           |  3 ++-
 turbolift_internals/src/kubernetes.rs    | 21 +++++++++++++++------
 turbolift_internals/src/local_queue.rs   |  5 ++++-
 turbolift_macros/Cargo.toml              |  2 +-
 turbolift_macros/src/lib.rs              |  8 +++++++-
 11 files changed, 38 insertions(+), 32 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index fc97981b..042ad1ee 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -17,3 +17,4 @@ turbolift_internals = { path = "./turbolift_internals" }
 chrono = { version = "0.4", optional = true }
 actix-web = { version = "3" }
 serde_json = { version = "1" }
+tokio-compat-02 = { version = "0.1" }
diff --git a/examples/kubernetes_example/Cargo.toml b/examples/kubernetes_example/Cargo.toml
index 694a3fc8..2f1a178f 100644
--- a/examples/kubernetes_example/Cargo.toml
+++ b/examples/kubernetes_example/Cargo.toml
@@ -9,7 +9,7 @@ edition = "2018"
 
 [dependencies]
 rand = "0.7"
-tokio = {version="0.2", features=["full"]}
+tokio = {version="0.3", features=["full"]}
 lazy_static = "1"
 futures = "0.3"
 cute = "0.3"
diff --git a/examples/kubernetes_example/src/main.rs b/examples/kubernetes_example/src/main.rs
index 100716d4..0899deec 100644
--- a/examples/kubernetes_example/src/main.rs
+++ b/examples/kubernetes_example/src/main.rs
@@ -26,11 +26,7 @@ fn random_numbers() -> Vec<u64> {
 fn main() {
     let input = random_numbers();
     let futures = c![square(*int), for int in &input];
-    let mut rt = tokio::runtime::Builder::new()
-        .threaded_scheduler()
-        .enable_all()
-        .build()
-        .unwrap();
+    let mut rt = tokio::runtime::Runtime::new().unwrap();
     let output = rt.block_on(try_join_all(futures)).unwrap();
     println!("input: {:?}\noutput: {:?}", input, output);
     if output != c![x*x, for x in input] {
@@ -46,11 +42,7 @@ mod tests {
     fn it_works() {
         let input = random_numbers();
         let futures = c![square(*int), for int in &input];
-        let mut rt = tokio::runtime::Builder::new()
-            .threaded_scheduler()
-            .enable_all()
-            .build()
-            .unwrap();
+        let mut rt = tokio::runtime::Runtime::new().unwrap();
         let output = rt.block_on(try_join_all(futures)).unwrap();
         assert_eq!(
             output,
diff --git a/examples/local_queue_example/Cargo.toml b/examples/local_queue_example/Cargo.toml
index 79681087..60935de1 100644
--- a/examples/local_queue_example/Cargo.toml
+++ b/examples/local_queue_example/Cargo.toml
@@ -12,7 +12,7 @@ edition = "2018"
 rand = "0.7"
 futures = "0.3"
 lazy_static = "1"
-tokio = {version="0.2", features=["full"]}
+tokio = {version="0.3", features=["full"]}
 turbolift = { path="../../" }
 
 # these are dependencies that need to be bundled into turbolift, but debugging how to reexport proc macros is hard.
diff --git a/examples/local_queue_example/src/main.rs b/examples/local_queue_example/src/main.rs
index 589df11e..f02bde47 100644
--- a/examples/local_queue_example/src/main.rs
+++ b/examples/local_queue_example/src/main.rs
@@ -26,11 +26,7 @@ fn main() {
         }
         v
     };
-    let mut rt = tokio::runtime::Builder::new()
-        .threaded_scheduler()
-        .enable_all()
-        .build()
-        .unwrap();
+    let mut rt = tokio::runtime::Runtime::new().unwrap();
     let output = rt.block_on(try_join_all(futures)).unwrap();
     println!("input: {:?}\noutput: {:?}", input, output);
     if output != input {
@@ -44,6 +40,7 @@ mod tests {
 
     #[test]
     fn it_works() {
+        println!("test started");
         let input = vec![rand::random(), rand::random(), rand::random()];
         let futures = {
             let mut v = Vec::new();
@@ -52,11 +49,7 @@ mod tests {
             }
             v
         };
-        let mut rt = tokio::runtime::Builder::new()
-            .threaded_scheduler()
-            .enable_all()
-            .build()
-            .unwrap();
+        let mut rt = tokio::runtime::Runtime::new().unwrap();
         let output = rt.block_on(try_join_all(futures)).unwrap();
         assert_eq!(input, output);
     }
diff --git a/src/lib.rs b/src/lib.rs
index ad75fc97..35443d1b 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -3,6 +3,7 @@ pub use chrono;
 
 pub use actix_web;
 pub use serde_json;
+pub use tokio_compat_02;
 
 pub use distributed_platform::{DistributionPlatform, DistributionResult};
 pub use turbolift_internals::*;
diff --git a/turbolift_internals/Cargo.toml b/turbolift_internals/Cargo.toml
index 8f05cd6d..bb167896 100644
--- a/turbolift_internals/Cargo.toml
+++ b/turbolift_internals/Cargo.toml
@@ -20,7 +20,8 @@ toml = "0.5"
 cargo-toml2 = "1"
 tempfile = "3.1"
 reqwest = {repository="https://github.com/DominicBurkart/reqwest"}
-tokio = { version = "0.2", features = ["full"] }
+tokio = { version = "0.3", features = ["full"] }
+tokio-compat-02 = "0.1"
 cute = "0.3"
 rand = "0.7"
 url = "2"
diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 49c3aafd..f7fe0860 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -9,6 +9,7 @@ use kube::api::{Api, PostParams};
 use kube::Client;
 use regex::Regex;
 use syn::export::Formatter;
+use tokio_compat_02::FutureExt;
 use url::Url;
 
 use crate::distributed_platform::{
@@ -83,9 +84,9 @@ fn function_to_deployment_name(function_name: &str) -> String {
 impl DistributionPlatform for K8s {
     async fn declare(&mut self, function_name: &str, project_tar: &[u8]) -> DistributionResult<()> {
         // connect to cluster. tries in-cluster configuration first, then falls back to kubeconfig file.
-        let deployment_client = Client::try_default().await?;
+        let deployment_client = Client::try_default().compat().await?;
         let deployments: Api<Deployment> = Api::namespaced(deployment_client, K8S_NAMESPACE);
-        let service_client = Client::try_default().await?;
+        let service_client = Client::try_default().compat().await?;
         let services: Api<Service> = Api::namespaced(service_client, K8S_NAMESPACE);
 
         // generate image & host it on a local registry
@@ -143,6 +144,7 @@ impl DistributionPlatform for K8s {
         println!("made deployment: {:?}", deployment);
         deployments
             .create(&PostParams::default(), &deployment)
+            .compat()
             .await?;
         println!("created deployment");
 
@@ -165,7 +167,10 @@ impl DistributionPlatform for K8s {
             }
         }))?;
         println!("made service");
-        let service = services.create(&PostParams::default(), &service).await?;
+        let service = services
+            .create(&PostParams::default(), &service)
+            .compat()
+            .await?;
         println!("created service");
         let service_ip = format!(
             "http://{}:5000",
@@ -213,8 +218,10 @@ impl DistributionPlatform for K8s {
             .request_client
             .get(query_url)
             .send()
+            .compat()
             .await?
             .text()
+            .compat()
             .await?);
         println!("dispatch returning: {:?}", resp);
         resp
@@ -364,11 +371,11 @@ CMD RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run --release 127.0.0.1:500
 impl Drop for K8s {
     fn drop(&mut self) {
         // delete the associated services and deployments from the functions we distributed
-        let mut rt = tokio::runtime::Runtime::new().unwrap();
+        let rt = tokio::runtime::Runtime::new().unwrap();
         rt.block_on(async {
-            let deployment_client = Client::try_default().await.unwrap();
+            let deployment_client = Client::try_default().compat().await.unwrap();
             let deployments: Api<Deployment> = Api::namespaced(deployment_client, K8S_NAMESPACE);
-            let service_client = Client::try_default().await.unwrap();
+            let service_client = Client::try_default().compat().await.unwrap();
             let services: Api<Service> = Api::namespaced(service_client, K8S_NAMESPACE);
 
             let distributed_functions = self.fn_names_to_services.keys();
@@ -376,11 +383,13 @@ impl Drop for K8s {
                 let service = function_to_service_name(function);
                 services
                     .delete(&service, &Default::default())
+                    .compat()
                     .await
                     .unwrap();
                 let deployment = function_to_deployment_name(function);
                 deployments
                     .delete(&deployment, &Default::default())
+                    .compat()
                     .await
                     .unwrap();
             }
diff --git a/turbolift_internals/src/local_queue.rs b/turbolift_internals/src/local_queue.rs
index e84aeb4c..e383abce 100644
--- a/turbolift_internals/src/local_queue.rs
+++ b/turbolift_internals/src/local_queue.rs
@@ -6,6 +6,7 @@ use std::time::Duration;
 
 use async_trait::async_trait;
 use std::process::{Child, Command};
+use tokio_compat_02::FutureExt;
 use url::Url;
 
 use crate::build_project::make_executable;
@@ -75,7 +76,7 @@ impl DistributionPlatform for LocalQueue {
                     .arg(&server_address_and_port_str)
                     .spawn()?;
                 println!("delaying");
-                tokio::time::delay_for(Duration::from_secs(60)).await;
+                tokio::time::sleep(Duration::from_secs(60)).await;
                 println!("delay completed");
                 // ^ sleep to make sure the server is initialized before continuing
                 self.fn_name_to_address
@@ -95,8 +96,10 @@ impl DistributionPlatform for LocalQueue {
             .request_client
             .get(query_url)
             .send()
+            .compat()
             .await?
             .text()
+            .compat()
             .await?);
         println!("dispatch returning: {:?}", resp);
         resp
diff --git a/turbolift_macros/Cargo.toml b/turbolift_macros/Cargo.toml
index 6160f5fe..a39d1eab 100644
--- a/turbolift_macros/Cargo.toml
+++ b/turbolift_macros/Cargo.toml
@@ -18,4 +18,4 @@ proc-macro2 = "1"
 fs_extra = "1"
 turbolift_internals = { path="../turbolift_internals" }
 futures = "0.3"
-cached = "0.19"
\ No newline at end of file
+cached = "0.19"
diff --git a/turbolift_macros/src/lib.rs b/turbolift_macros/src/lib.rs
index ed85e02c..dc6d265e 100644
--- a/turbolift_macros/src/lib.rs
+++ b/turbolift_macros/src/lib.rs
@@ -48,6 +48,7 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
     // todo make code below hygienic in case sanitized_file also imports from actix_web
     let main_file = q! {
         use turbolift::actix_web::{self, get, web, HttpResponse, Result};
+        use turbolift::tokio_compat_02::FutureExt;
 
         #sanitized_file
         #dummy_function
@@ -76,6 +77,7 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
             )
             .bind(ip_and_port)?
             .run()
+            .compat()
             .await
         }
     };
@@ -162,6 +164,7 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
             use std::time::Duration;
             use turbolift::distributed_platform::DistributionPlatform;
             use turbolift::DistributionResult;
+            use turbolift::tokio_compat_02::FutureExt;
 
             let mut platform = #distribution_platform.lock()?;
 
@@ -169,6 +172,7 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
                 println!("launching declare");
                 platform
                     .declare(#original_target_function_name, #project_source_binary)
+                    .compat()
                     .await?;
                 println!("declare completed");
             }
@@ -180,7 +184,9 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
                 .dispatch(
                     #original_target_function_name,
                     params.to_string()
-                ).await?;
+                )
+                .compat()
+                .await?;
             println!("dispatch completed");
             Ok(turbolift::serde_json::from_str(&resp_string)?)
         }

From 49dffbacc0a0e8fa36a24d23ebe4208eb0265722 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Mon, 26 Oct 2020 20:15:59 +0100
Subject: [PATCH 055/145] don't capture stdout in tests

---
 examples/kubernetes_example/example_runtime.sh | 6 +++---
 examples/local_queue_example/Dockerfile        | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/examples/kubernetes_example/example_runtime.sh b/examples/kubernetes_example/example_runtime.sh
index 7e457306..0cde2b07 100644
--- a/examples/kubernetes_example/example_runtime.sh
+++ b/examples/kubernetes_example/example_runtime.sh
@@ -6,18 +6,18 @@ set -e
 cd turbolift/examples/kubernetes_example
 
 # run non-distributed example without cluster in environment
-RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test
+RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test -- --nocapture
 RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run
 
 # setup cluster (will be used in all tests & runs)
 ../../../kind create cluster
 
 # run same tests with cluster in environment
-RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test
+RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test -- --nocapture
 RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run
 
 # run distributed tests
-RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test --features distributed
+RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test --features distributed -- --nocapture
 RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run --features distributed
 
 ../../../kind delete cluster
diff --git a/examples/local_queue_example/Dockerfile b/examples/local_queue_example/Dockerfile
index 3f792a7d..b8d94e70 100644
--- a/examples/local_queue_example/Dockerfile
+++ b/examples/local_queue_example/Dockerfile
@@ -4,9 +4,9 @@ COPY ./ turbolift
 WORKDIR turbolift/examples/local_queue_example
 
 # test
-RUN RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test
-RUN RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test --features distributed
+RUN RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test -- --nocapture
+RUN RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test --features distributed -- --nocapture
 
 # run
 RUN RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run
-RUN RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run --features distributed
\ No newline at end of file
+RUN RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run --features distributed

From d6cc7a30fafd3233a41ce008b646e54e908290a9 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Mon, 26 Oct 2020 20:16:21 +0100
Subject: [PATCH 056/145] add debug comments

---
 turbolift_macros/src/lib.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/turbolift_macros/src/lib.rs b/turbolift_macros/src/lib.rs
index dc6d265e..3d3d9118 100644
--- a/turbolift_macros/src/lib.rs
+++ b/turbolift_macros/src/lib.rs
@@ -166,8 +166,12 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
             use turbolift::DistributionResult;
             use turbolift::tokio_compat_02::FutureExt;
 
+            println!("in original target function");
+
             let mut platform = #distribution_platform.lock()?;
 
+            println!("platform generated");
+
             if !platform.has_declared(#original_target_function_name) {
                 println!("launching declare");
                 platform

From 93a895e9ef54f001d63e12437399f5d9415759b3 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Tue, 27 Oct 2020 11:59:43 +0100
Subject: [PATCH 057/145] use tokio mutex because std sync mutex hangs on lock
 in our use case

---
 examples/kubernetes_example/src/main.rs  | 4 ++--
 examples/local_queue_example/src/main.rs | 3 +--
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/examples/kubernetes_example/src/main.rs b/examples/kubernetes_example/src/main.rs
index 0899deec..fa733850 100644
--- a/examples/kubernetes_example/src/main.rs
+++ b/examples/kubernetes_example/src/main.rs
@@ -1,11 +1,11 @@
-use std::sync::Mutex;
-
 #[macro_use]
 extern crate lazy_static;
 #[macro_use(c)]
 extern crate cute;
 use futures::future::try_join_all;
 use rand::{thread_rng, Rng};
+use tokio::sync::Mutex;
+
 use turbolift::kubernetes::K8s;
 use turbolift::on;
 
diff --git a/examples/local_queue_example/src/main.rs b/examples/local_queue_example/src/main.rs
index f02bde47..c277be43 100644
--- a/examples/local_queue_example/src/main.rs
+++ b/examples/local_queue_example/src/main.rs
@@ -1,5 +1,3 @@
-use std::sync::Mutex;
-
 extern crate proc_macro;
 use futures::future::try_join_all;
 use rand;
@@ -7,6 +5,7 @@ use turbolift::local_queue::LocalQueue;
 use turbolift::on;
 #[macro_use]
 extern crate lazy_static;
+use tokio::sync::Mutex;
 
 lazy_static! {
     static ref LOCAL: Mutex<LocalQueue> = Mutex::new(LocalQueue::new());

From ea44eecb46bf5b74a52fab87a321840080d2abff Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Tue, 27 Oct 2020 12:04:49 +0100
Subject: [PATCH 058/145] allow lint errors

---
 turbolift_macros/src/lib.rs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/turbolift_macros/src/lib.rs b/turbolift_macros/src/lib.rs
index 3d3d9118..c970c92d 100644
--- a/turbolift_macros/src/lib.rs
+++ b/turbolift_macros/src/lib.rs
@@ -122,7 +122,9 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
     .expect("error editing cargo file");
 
     // lint project
-    build_project::lint(&function_cache_proj_path).expect("linting error");
+    if let Err(e) = build_project::lint(&function_cache_proj_path) {
+        eprintln!("ignoring linting error: {:?}", e)
+    }
 
     // check project and give errors
     build_project::check(&function_cache_proj_path).expect("error checking function");
@@ -168,7 +170,7 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
 
             println!("in original target function");
 
-            let mut platform = #distribution_platform.lock()?;
+            let mut platform = #distribution_platform.lock().await;
 
             println!("platform generated");
 

From 0f43e5b30eacd9b4cac9ff211c131c74164dc4cd Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Tue, 27 Oct 2020 17:26:01 +0100
Subject: [PATCH 059/145] make kind wait until the control pane reaches a ready
 status (or times out)

---
 examples/kubernetes_example/example_runtime.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/kubernetes_example/example_runtime.sh b/examples/kubernetes_example/example_runtime.sh
index 0cde2b07..3674ae17 100644
--- a/examples/kubernetes_example/example_runtime.sh
+++ b/examples/kubernetes_example/example_runtime.sh
@@ -10,7 +10,7 @@ RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test -- --nocapture
 RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run
 
 # setup cluster (will be used in all tests & runs)
-../../../kind create cluster
+../../../kind create cluster --wait 20m
 
 # run same tests with cluster in environment
 RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test -- --nocapture

From 5f64b7abe41484058c99a904a071b02a2be5f71b Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Tue, 27 Oct 2020 17:35:29 +0100
Subject: [PATCH 060/145] update comment in kubernetes example

---
 examples/kubernetes_example/example_runtime.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/kubernetes_example/example_runtime.sh b/examples/kubernetes_example/example_runtime.sh
index 3674ae17..7244a398 100644
--- a/examples/kubernetes_example/example_runtime.sh
+++ b/examples/kubernetes_example/example_runtime.sh
@@ -12,7 +12,7 @@ RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run
 # setup cluster (will be used in all tests & runs)
 ../../../kind create cluster --wait 20m
 
-# run same tests with cluster in environment
+# run non-distributed tests again with cluster in environment
 RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test -- --nocapture
 RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run
 

From b8538cea5a655a1db814ea57c65198efa044e5d7 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Wed, 28 Oct 2020 08:01:31 +0100
Subject: [PATCH 061/145] alias kind

---
 examples/kubernetes_example/example_runtime.sh | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/examples/kubernetes_example/example_runtime.sh b/examples/kubernetes_example/example_runtime.sh
index 7244a398..290dd9b7 100644
--- a/examples/kubernetes_example/example_runtime.sh
+++ b/examples/kubernetes_example/example_runtime.sh
@@ -3,6 +3,9 @@
 # error if any command fails
 set -e
 
+# shellcheck disable=SC2139
+alias kind="$PWD/kind"
+
 cd turbolift/examples/kubernetes_example
 
 # run non-distributed example without cluster in environment
@@ -10,7 +13,7 @@ RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test -- --nocapture
 RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run
 
 # setup cluster (will be used in all tests & runs)
-../../../kind create cluster --wait 20m
+kind create cluster --wait 20m
 
 # run non-distributed tests again with cluster in environment
 RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test -- --nocapture
@@ -20,4 +23,4 @@ RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run
 RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test --features distributed -- --nocapture
 RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run --features distributed
 
-../../../kind delete cluster
+kind delete cluster

From af966ca46b4cc3d479d945cc4aae118d1cb5ab2f Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Wed, 28 Oct 2020 08:23:24 +0100
Subject: [PATCH 062/145] k8s: use default namespace

---
 turbolift_internals/src/kubernetes.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index f7fe0860..26c71516 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -17,7 +17,7 @@ use crate::distributed_platform::{
 };
 use crate::utils::get_open_socket;
 
-const K8S_NAMESPACE: &str = "turbolift";
+const K8S_NAMESPACE: &str = "default";
 type ImageTag = String;
 
 /// `K8s` is the interface for turning rust functions into autoscaling microservices

From 7c0a5ea5ea52f3f453fcb02bc28ef3deafd99915 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Wed, 28 Oct 2020 08:54:35 +0100
Subject: [PATCH 063/145] change variable name

---
 turbolift_internals/src/kubernetes.rs | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 26c71516..d58ce9e6 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -17,7 +17,7 @@ use crate::distributed_platform::{
 };
 use crate::utils::get_open_socket;
 
-const K8S_NAMESPACE: &str = "default";
+const TURBOLIFT_K8S_NAMESPACE: &str = "default";
 type ImageTag = String;
 
 /// `K8s` is the interface for turning rust functions into autoscaling microservices
@@ -85,9 +85,10 @@ impl DistributionPlatform for K8s {
     async fn declare(&mut self, function_name: &str, project_tar: &[u8]) -> DistributionResult<()> {
         // connect to cluster. tries in-cluster configuration first, then falls back to kubeconfig file.
         let deployment_client = Client::try_default().compat().await?;
-        let deployments: Api<Deployment> = Api::namespaced(deployment_client, K8S_NAMESPACE);
+        let deployments: Api<Deployment> =
+            Api::namespaced(deployment_client, TURBOLIFT_K8S_NAMESPACE);
         let service_client = Client::try_default().compat().await?;
-        let services: Api<Service> = Api::namespaced(service_client, K8S_NAMESPACE);
+        let services: Api<Service> = Api::namespaced(service_client, TURBOLIFT_K8S_NAMESPACE);
 
         // generate image & host it on a local registry
         let registry_url = setup_registry(function_name, project_tar)?;
@@ -374,9 +375,10 @@ impl Drop for K8s {
         let rt = tokio::runtime::Runtime::new().unwrap();
         rt.block_on(async {
             let deployment_client = Client::try_default().compat().await.unwrap();
-            let deployments: Api<Deployment> = Api::namespaced(deployment_client, K8S_NAMESPACE);
+            let deployments: Api<Deployment> =
+                Api::namespaced(deployment_client, TURBOLIFT_K8S_NAMESPACE);
             let service_client = Client::try_default().compat().await.unwrap();
-            let services: Api<Service> = Api::namespaced(service_client, K8S_NAMESPACE);
+            let services: Api<Service> = Api::namespaced(service_client, TURBOLIFT_K8S_NAMESPACE);
 
             let distributed_functions = self.fn_names_to_services.keys();
             for function in distributed_functions {

From 8516277eb8b1880bcfb3c798d4094a8bfc027a3f Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Wed, 28 Oct 2020 09:00:13 +0100
Subject: [PATCH 064/145] add debug statements to kubernetes example

---
 examples/kubernetes_example/example_runtime.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/examples/kubernetes_example/example_runtime.sh b/examples/kubernetes_example/example_runtime.sh
index 290dd9b7..910f466e 100644
--- a/examples/kubernetes_example/example_runtime.sh
+++ b/examples/kubernetes_example/example_runtime.sh
@@ -14,6 +14,8 @@ RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run
 
 # setup cluster (will be used in all tests & runs)
 kind create cluster --wait 20m
+kubectl cluster-info --context kind-kind
+kubectl get ns
 
 # run non-distributed tests again with cluster in environment
 RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test -- --nocapture

From 6091eeb034397cb8e00e3f1571f957778b3caeb0 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Wed, 28 Oct 2020 09:20:15 +0100
Subject: [PATCH 065/145] install kubectl in k8s example environment

---
 examples/kubernetes_example/Dockerfile         | 4 +++-
 examples/kubernetes_example/example_runtime.sh | 2 ++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/examples/kubernetes_example/Dockerfile b/examples/kubernetes_example/Dockerfile
index b2540871..54f4a9f4 100644
--- a/examples/kubernetes_example/Dockerfile
+++ b/examples/kubernetes_example/Dockerfile
@@ -9,5 +9,7 @@ RUN  apt-get update \
   && rm -rf /var/lib/apt/lists/*
 RUN wget -O ./kind --show-progress "https://kind.sigs.k8s.io/dl/v0.9.0/kind-$(uname)-amd64"
 RUN chmod +x ./kind
+RUN wget -O ./kubectl --show-progress "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl"
+RUN chmod +x ./kubectl
 COPY ./ turbolift
-CMD sh turbolift/examples/kubernetes_example/example_runtime.sh
\ No newline at end of file
+CMD sh turbolift/examples/kubernetes_example/example_runtime.sh
diff --git a/examples/kubernetes_example/example_runtime.sh b/examples/kubernetes_example/example_runtime.sh
index 910f466e..9ab40c25 100644
--- a/examples/kubernetes_example/example_runtime.sh
+++ b/examples/kubernetes_example/example_runtime.sh
@@ -5,6 +5,8 @@ set -e
 
 # shellcheck disable=SC2139
 alias kind="$PWD/kind"
+# shellcheck disable=SC2139
+alias kubectl="$PWD/kubectl"
 
 cd turbolift/examples/kubernetes_example
 

From 5b6c3a7f713d361f3d69eae6877da241f39e5712 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Wed, 28 Oct 2020 09:44:34 +0100
Subject: [PATCH 066/145] add debug statements to kubernetes example

---
 examples/kubernetes_example/Dockerfile         | 4 ++--
 examples/kubernetes_example/example_runtime.sh | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/examples/kubernetes_example/Dockerfile b/examples/kubernetes_example/Dockerfile
index 54f4a9f4..17654a8a 100644
--- a/examples/kubernetes_example/Dockerfile
+++ b/examples/kubernetes_example/Dockerfile
@@ -7,9 +7,9 @@ RUN  apt-get update \
   && apt-get install -y wget \
   && apt-get install -y docker.io \
   && rm -rf /var/lib/apt/lists/*
-RUN wget -O ./kind --show-progress "https://kind.sigs.k8s.io/dl/v0.9.0/kind-$(uname)-amd64"
-RUN chmod +x ./kind
 RUN wget -O ./kubectl --show-progress "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl"
 RUN chmod +x ./kubectl
+RUN wget -O ./kind --show-progress "https://kind.sigs.k8s.io/dl/v0.9.0/kind-$(uname)-amd64"
+RUN chmod +x ./kind
 COPY ./ turbolift
 CMD sh turbolift/examples/kubernetes_example/example_runtime.sh
diff --git a/examples/kubernetes_example/example_runtime.sh b/examples/kubernetes_example/example_runtime.sh
index 9ab40c25..71200f4a 100644
--- a/examples/kubernetes_example/example_runtime.sh
+++ b/examples/kubernetes_example/example_runtime.sh
@@ -16,7 +16,7 @@ RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run
 
 # setup cluster (will be used in all tests & runs)
 kind create cluster --wait 20m
-kubectl cluster-info --context kind-kind
+kubectl cluster-info --context kind-kind || echo context setting failed && kubectl cluster-info dump
 kubectl get ns
 
 # run non-distributed tests again with cluster in environment

From 663e56f331d52d58b70c184f95faa3e013a0e481 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Wed, 28 Oct 2020 14:30:42 +0100
Subject: [PATCH 067/145] group installation commands in k8s dockerfile

---
 examples/kubernetes_example/Dockerfile | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/examples/kubernetes_example/Dockerfile b/examples/kubernetes_example/Dockerfile
index 17654a8a..60b90b72 100644
--- a/examples/kubernetes_example/Dockerfile
+++ b/examples/kubernetes_example/Dockerfile
@@ -4,8 +4,7 @@
 # set up kind (k8s in docker)
 FROM rustlang/rust:nightly
 RUN  apt-get update \
-  && apt-get install -y wget \
-  && apt-get install -y docker.io \
+  && apt-get install -y wget docker.io \
   && rm -rf /var/lib/apt/lists/*
 RUN wget -O ./kubectl --show-progress "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl"
 RUN chmod +x ./kubectl

From e7128081f7bb624c00a0fcb5461124c1a26b2b40 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Thu, 29 Oct 2020 10:23:15 +0100
Subject: [PATCH 068/145] refactor CI and testing schema

---
 .github/workflows/docker.yml                  | 23 ------------
 .github/workflows/examples.yml                | 36 +++++++++++++++++++
 .github/workflows/{rust.yml => linting.yml}   |  4 +--
 examples/kubernetes_example/Dockerfile        | 14 --------
 .../kubernetes_example/example_runtime.sh     | 30 ----------------
 .../run_distributed_tests.sh                  |  8 +++++
 .../run_non_distributed_tests.sh              |  8 +++++
 7 files changed, 54 insertions(+), 69 deletions(-)
 delete mode 100644 .github/workflows/docker.yml
 create mode 100644 .github/workflows/examples.yml
 rename .github/workflows/{rust.yml => linting.yml} (96%)
 delete mode 100644 examples/kubernetes_example/Dockerfile
 delete mode 100644 examples/kubernetes_example/example_runtime.sh
 create mode 100644 examples/kubernetes_example/run_distributed_tests.sh
 create mode 100644 examples/kubernetes_example/run_non_distributed_tests.sh

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
deleted file mode 100644
index 48367826..00000000
--- a/.github/workflows/docker.yml
+++ /dev/null
@@ -1,23 +0,0 @@
-name: docker
-
-on:
-  push:
-    branches: master
-  pull_request:
-    branches: [ master ]
-
-jobs:
-  local_queue_example:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: build local queue example
-        run: docker build -f examples/local_queue_example/Dockerfile .
-  kubernetes_example:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: build k8s example
-        run: docker build -f examples/kubernetes_example/Dockerfile -t kub .
-      - name: run k8s example
-        run: docker run -v /var/run/docker.sock:/var/run/docker.sock kub
\ No newline at end of file
diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml
new file mode 100644
index 00000000..544f3584
--- /dev/null
+++ b/.github/workflows/examples.yml
@@ -0,0 +1,36 @@
+name: examples
+
+on:
+  push:
+    branches: master
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  local_queue_example:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: build local queue example
+        run: docker build -f examples/local_queue_example/Dockerfile .
+
+  kubernetes_example:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          path: './turbolift'
+      - name: install rustup and rust nightly
+        run: |
+            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+            rustup toolchain install nightly
+      - name: run non-distributed example without cluster env
+        run: |
+            cd turbolift/examples/kubernetes_example
+            sh run_non_distributed_tests.sh
+      - uses: engineerd/setup-kind@v0.4.0
+      - name: run distributed and non-distributed examples
+        run: |
+          cd turbolift/examples/kubernetes_example
+          sh run_non_distributed_tests.sh
+          sh run_distributed_tests.sh
diff --git a/.github/workflows/rust.yml b/.github/workflows/linting.yml
similarity index 96%
rename from .github/workflows/rust.yml
rename to .github/workflows/linting.yml
index ef78c20a..5f955bf8 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/linting.yml
@@ -1,8 +1,8 @@
-name: rust
+name: linting
 
 on:
   push:
-    branches: [ master ]
+    branches: master
   pull_request:
     branches: [ master ]
 
diff --git a/examples/kubernetes_example/Dockerfile b/examples/kubernetes_example/Dockerfile
deleted file mode 100644
index 60b90b72..00000000
--- a/examples/kubernetes_example/Dockerfile
+++ /dev/null
@@ -1,14 +0,0 @@
-# docker build -f examples/kubernetes_example/Dockerfile -t kub .
-# docker run -v /var/run/docker.sock:/var/run/docker.sock kub
-
-# set up kind (k8s in docker)
-FROM rustlang/rust:nightly
-RUN  apt-get update \
-  && apt-get install -y wget docker.io \
-  && rm -rf /var/lib/apt/lists/*
-RUN wget -O ./kubectl --show-progress "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl"
-RUN chmod +x ./kubectl
-RUN wget -O ./kind --show-progress "https://kind.sigs.k8s.io/dl/v0.9.0/kind-$(uname)-amd64"
-RUN chmod +x ./kind
-COPY ./ turbolift
-CMD sh turbolift/examples/kubernetes_example/example_runtime.sh
diff --git a/examples/kubernetes_example/example_runtime.sh b/examples/kubernetes_example/example_runtime.sh
deleted file mode 100644
index 71200f4a..00000000
--- a/examples/kubernetes_example/example_runtime.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/usr/bin/env sh
-
-# error if any command fails
-set -e
-
-# shellcheck disable=SC2139
-alias kind="$PWD/kind"
-# shellcheck disable=SC2139
-alias kubectl="$PWD/kubectl"
-
-cd turbolift/examples/kubernetes_example
-
-# run non-distributed example without cluster in environment
-RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test -- --nocapture
-RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run
-
-# setup cluster (will be used in all tests & runs)
-kind create cluster --wait 20m
-kubectl cluster-info --context kind-kind || echo context setting failed && kubectl cluster-info dump
-kubectl get ns
-
-# run non-distributed tests again with cluster in environment
-RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test -- --nocapture
-RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run
-
-# run distributed tests
-RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test --features distributed -- --nocapture
-RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run --features distributed
-
-kind delete cluster
diff --git a/examples/kubernetes_example/run_distributed_tests.sh b/examples/kubernetes_example/run_distributed_tests.sh
new file mode 100644
index 00000000..0e4038e6
--- /dev/null
+++ b/examples/kubernetes_example/run_distributed_tests.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env sh
+
+# error if any command fails
+set -e
+
+# run distributed tests
+RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test --features distributed -- --nocapture
+RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run --features distributed
diff --git a/examples/kubernetes_example/run_non_distributed_tests.sh b/examples/kubernetes_example/run_non_distributed_tests.sh
new file mode 100644
index 00000000..82a7d058
--- /dev/null
+++ b/examples/kubernetes_example/run_non_distributed_tests.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env sh
+
+# error if any command fails
+set -e
+
+# run non-distributed tests
+RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test -- --nocapture
+RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run

From e5132b4585ae464fcfb34917d99f0a11c3a63289 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Thu, 29 Oct 2020 12:10:28 +0100
Subject: [PATCH 069/145] use a valid deployment name (only alphanumerics +
 '-')

---
 turbolift_internals/src/kubernetes.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index d58ce9e6..386ba736 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -73,11 +73,11 @@ impl std::fmt::Display for AutoscaleError {
 }
 
 fn function_to_service_name(function_name: &str) -> String {
-    function_name.to_string() + "-service"
+    function_name.to_string().replace("_", "-") + "-service"
 }
 
 fn function_to_deployment_name(function_name: &str) -> String {
-    function_name.to_string() + "-deployment"
+    function_name.to_string().replace("_", "-") + "-deployment"
 }
 
 #[async_trait]
@@ -100,7 +100,7 @@ impl DistributionPlatform for K8s {
         let deployment_name = function_to_deployment_name(function_name);
         let service_name = function_to_service_name(function_name);
         println!("got service_name");
-        let app_name = function_name.to_string();
+        let app_name = function_name.to_string().replace("_", "-");
         println!("... app_name is fine...");
         let deployment_json = serde_json::json!({
             "apiVersion": "apps/v1",

From 9b4d31a6223ff0d0e7299ba7fbb816f803e9aeb0 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Thu, 29 Oct 2020 16:45:38 +0100
Subject: [PATCH 070/145] use valid container name (only alphanumerics + '-')

---
 turbolift_internals/src/kubernetes.rs | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 386ba736..63a20ce8 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -80,6 +80,14 @@ fn function_to_deployment_name(function_name: &str) -> String {
     function_name.to_string().replace("_", "-") + "-deployment"
 }
 
+fn function_to_app_name(function_name: &str) -> String {
+    function_name.to_string().replace("_", "-")
+}
+
+fn function_to_container_name(function_name: &str) -> String {
+    function_name.to_string().replace("_", "-") + "-container"
+}
+
 #[async_trait]
 impl DistributionPlatform for K8s {
     async fn declare(&mut self, function_name: &str, project_tar: &[u8]) -> DistributionResult<()> {
@@ -100,7 +108,8 @@ impl DistributionPlatform for K8s {
         let deployment_name = function_to_deployment_name(function_name);
         let service_name = function_to_service_name(function_name);
         println!("got service_name");
-        let app_name = function_name.to_string().replace("_", "-");
+        let app_name = function_to_app_name(function_name);
+        let container_name = function_to_container_name(function_name);
         println!("... app_name is fine...");
         let deployment_json = serde_json::json!({
             "apiVersion": "apps/v1",
@@ -127,7 +136,7 @@ impl DistributionPlatform for K8s {
                     "spec": {
                         "containers": [
                             {
-                                "name": tag_in_reg,
+                                "name": container_name,
                                 "image": image_url,
                                 "ports": [
                                     {

From ca7573d1720544d7ca38ab78f047241698cc2ad0 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Thu, 29 Oct 2020 18:27:30 +0100
Subject: [PATCH 071/145] bug fix: ports accepts an array, not a map

---
 turbolift_internals/src/kubernetes.rs | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 63a20ce8..9339ee7e 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -169,11 +169,13 @@ impl DistributionPlatform for K8s {
                 "selector": {
                     "app": deployment_name
                 },
-                "ports": {
-                    "protocol": "HTTP",
-                    "port": 5000,
-                    "targetPort": 5000
-                }
+                "ports": [
+                    {
+                        "protocol": "HTTP",
+                        "port": 5000,
+                        "targetPort": 5000
+                    }
+                ]
             }
         }))?;
         println!("made service");

From 570cf571a1cd715b461e611ba59b50a9748c1f85 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Thu, 29 Oct 2020 19:03:28 +0100
Subject: [PATCH 072/145] dont specify invalid port protocol

---
 turbolift_internals/src/kubernetes.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 9339ee7e..19264a3e 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -171,7 +171,6 @@ impl DistributionPlatform for K8s {
                 },
                 "ports": [
                     {
-                        "protocol": "HTTP",
                         "port": 5000,
                         "targetPort": 5000
                     }

From 811c16a0d32d54f783b0a748e466d5086f985998 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Thu, 29 Oct 2020 20:33:44 +0100
Subject: [PATCH 073/145] use nodeport to expose service to network

---
 turbolift_internals/src/kubernetes.rs | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 19264a3e..03049fa6 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -166,6 +166,7 @@ impl DistributionPlatform for K8s {
                 "name": service_name
             },
             "spec": {
+                "type": "NodePort",
                 "selector": {
                     "app": deployment_name
                 },
@@ -184,12 +185,16 @@ impl DistributionPlatform for K8s {
             .await?;
         println!("created service");
         let service_ip = format!(
-            "http://{}:5000",
+            "http://localhost:{}",
             service
                 .spec
                 .expect("no specification found for service")
-                .cluster_ip
-                .expect("no cluster ip found for service")
+                .ports
+                .expect("no ports found for service")
+                .iter()
+                .filter_map(|port| port.node_port)
+                .next()
+                .expect("no node port assigned to service")
         );
         println!("service_ip {}", service_ip);
 

From 886ed0da56680c4c0abbc80174e838ddba907c82 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Fri, 30 Oct 2020 09:24:54 +0100
Subject: [PATCH 074/145] expose service to the device running each node

---
 turbolift_internals/src/kubernetes.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 03049fa6..acb240ed 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -175,6 +175,9 @@ impl DistributionPlatform for K8s {
                         "port": 5000,
                         "targetPort": 5000
                     }
+                ],
+                "externalIPs": [
+                    "127.0.0.1"
                 ]
             }
         }))?;
@@ -185,7 +188,7 @@ impl DistributionPlatform for K8s {
             .await?;
         println!("created service");
         let service_ip = format!(
-            "http://localhost:{}",
+            "http://127.0.0.1:{}",
             service
                 .spec
                 .expect("no specification found for service")

From 8d5cb30c188b90cccddeee4e6e07d12b2bd19926 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Fri, 30 Oct 2020 10:31:37 +0100
Subject: [PATCH 075/145] add script for running k8s tests locally

---
 .../kubernetes_example/run_tests_locally.sh   | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)
 create mode 100644 examples/kubernetes_example/run_tests_locally.sh

diff --git a/examples/kubernetes_example/run_tests_locally.sh b/examples/kubernetes_example/run_tests_locally.sh
new file mode 100644
index 00000000..cba8ac36
--- /dev/null
+++ b/examples/kubernetes_example/run_tests_locally.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env sh
+
+# assumes cargo, rust nightly, kind, and kubectl are installed. run from turbolift/examples/kubernetes_example
+
+# error if any command fails
+set -e
+
+# run non-distributed tests without cluster
+. ./run_non_distributed_tests.sh
+
+# generate cluster
+kind create cluster
+
+# re-run non-distributed tests
+. ./run_non_distributed_tests.sh || kind delete cluster
+
+# run distributed tests
+. ./run_distributed_tests.sh || kind delete cluster
+
+# delete cluster
+kind delete cluster

From 306d117dde35d237dd781bfb6da9ef4c426b5119 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Mon, 2 Nov 2020 15:08:00 +0100
Subject: [PATCH 076/145] refactor: don't try to set up a local registry the
 user has to set up a local registry in advance that is connected to their
 cluster. By default, k8s has no way of accessing a local registry. So, don't
 try to set up a local registry, and instead assume that it's running on some
 arbitrary port (hardcoded here as port 5000).

---
 .../kubernetes_example/run_tests_locally.sh   |   8 +-
 examples/kubernetes_example/setup_kind.sh     |  42 +++++
 turbolift_internals/src/kubernetes.rs         | 174 ++++++------------
 turbolift_internals/src/utils.rs              |   6 -
 4 files changed, 107 insertions(+), 123 deletions(-)
 create mode 100644 examples/kubernetes_example/setup_kind.sh

diff --git a/examples/kubernetes_example/run_tests_locally.sh b/examples/kubernetes_example/run_tests_locally.sh
index cba8ac36..1887ffaa 100644
--- a/examples/kubernetes_example/run_tests_locally.sh
+++ b/examples/kubernetes_example/run_tests_locally.sh
@@ -9,13 +9,13 @@ set -e
 . ./run_non_distributed_tests.sh
 
 # generate cluster
-kind create cluster
+. ./setup_kind.sh || kind delete
 
 # re-run non-distributed tests
-. ./run_non_distributed_tests.sh || kind delete cluster
+. ./run_non_distributed_tests.sh || kind delete
 
 # run distributed tests
-. ./run_distributed_tests.sh || kind delete cluster
+. ./run_distributed_tests.sh || kind delete
 
 # delete cluster
-kind delete cluster
+kind delete
diff --git a/examples/kubernetes_example/setup_kind.sh b/examples/kubernetes_example/setup_kind.sh
new file mode 100644
index 00000000..b72cc66a
--- /dev/null
+++ b/examples/kubernetes_example/setup_kind.sh
@@ -0,0 +1,42 @@
+#!/bin/sh
+# from https://kind.sigs.k8s.io/docs/user/local-registry/
+
+set -o errexit
+
+# create registry container unless it already exists
+reg_name='kind-registry'
+reg_port='5000'
+running="$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)"
+if [ "${running}" != 'true' ]; then
+  docker run \
+    -d --restart=always -p "${reg_port}:5000" --name "${reg_name}" \
+    registry:2
+fi
+
+# create a cluster with the local registry enabled in containerd
+cat <<EOF | kind create cluster --config=-
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+containerdConfigPatches:
+- |-
+  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:${reg_port}"]
+    endpoint = ["http://${reg_name}:${reg_port}"]
+EOF
+
+# connect the registry to the cluster network
+# (the network may already be connected)
+docker network connect "kind" "${reg_name}" || true
+
+# Document the local registry
+# https://github.com/kubernetes/enhancements/tree/master/keps/sig-cluster-lifecycle/generic/1755-communicating-a-local-registry
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: local-registry-hosting
+  namespace: kube-public
+data:
+  localRegistryHosting.v1: |
+    host: "localhost:${reg_port}"
+    help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
+EOF
diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index acb240ed..ee71d028 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -15,9 +15,9 @@ use url::Url;
 use crate::distributed_platform::{
     ArgsString, DistributionPlatform, DistributionResult, JsonResponse,
 };
-use crate::utils::get_open_socket;
 
 const TURBOLIFT_K8S_NAMESPACE: &str = "default";
+const LOCAL_REGISTRY_URL: &str = "http://localhost:5000";
 type ImageTag = String;
 
 /// `K8s` is the interface for turning rust functions into autoscaling microservices
@@ -99,11 +99,10 @@ impl DistributionPlatform for K8s {
         let services: Api<Service> = Api::namespaced(service_client, TURBOLIFT_K8S_NAMESPACE);
 
         // generate image & host it on a local registry
-        let registry_url = setup_registry(function_name, project_tar)?;
+        let registry_url = Url::parse(LOCAL_REGISTRY_URL)?;
         let tag_in_reg = make_image(function_name, project_tar, &registry_url)?;
         let image_url = registry_url.join(&tag_in_reg)?.as_str().to_string();
 
-        // make deployment
         println!("wooo");
         let deployment_name = function_to_deployment_name(function_name);
         let service_name = function_to_service_name(function_name);
@@ -111,6 +110,8 @@ impl DistributionPlatform for K8s {
         let app_name = function_to_app_name(function_name);
         let container_name = function_to_container_name(function_name);
         println!("... app_name is fine...");
+
+        // make deployment
         let deployment_json = serde_json::json!({
             "apiVersion": "apps/v1",
             "kind": "Deployment",
@@ -172,12 +173,9 @@ impl DistributionPlatform for K8s {
                 },
                 "ports": [
                     {
-                        "port": 5000,
-                        "targetPort": 5000
+                        "protocol": "TCP",
+                        "port": 5000
                     }
-                ],
-                "externalIPs": [
-                    "127.0.0.1"
                 ]
             }
         }))?;
@@ -187,18 +185,16 @@ impl DistributionPlatform for K8s {
             .compat()
             .await?;
         println!("created service");
-        let service_ip = format!(
-            "http://127.0.0.1:{}",
-            service
-                .spec
-                .expect("no specification found for service")
-                .ports
-                .expect("no ports found for service")
-                .iter()
-                .filter_map(|port| port.node_port)
-                .next()
-                .expect("no node port assigned to service")
-        );
+        let node_port = service
+            .spec
+            .expect("no specification found for service")
+            .ports
+            .expect("no ports found for service")
+            .iter()
+            .filter_map(|port| port.node_port)
+            .next()
+            .expect("no node port assigned to service");
+        let service_ip = format!("http://localhost:{}", node_port);
         println!("service_ip {}", service_ip);
 
         // todo make sure that the pod and service were correctly started before returning
@@ -255,59 +251,6 @@ lazy_static! {
     static ref PORT_RE: Regex = Regex::new(r"0\.0\.0\.0:(\d+)->").unwrap();
 }
 
-fn setup_registry(_function_name: &str, _project_tar: &[u8]) -> anyhow::Result<Url> {
-    // check if registry is already running locally
-    let ps_output = Command::new("docker")
-        .args("ps -f name=turbolift-registry".split(' '))
-        .output()?;
-    if !ps_output.status.success() {
-        return Err(anyhow::anyhow!(
-            "docker ps failed. Is the docker daemon running?"
-        ));
-    }
-    let utf8 = String::from_utf8(ps_output.stdout)?;
-    let num_lines = utf8.as_str().lines().count();
-    let port = if num_lines == 1 {
-        // registry not running. Start the registry.
-        let port = get_open_socket()?.local_addr()?.port().to_string(); // todo hack
-        let args_str = format!(
-            "run -d -p {}:5000 --restart=always --name turbolift-registry registry:2",
-            port
-        );
-        let status = Command::new("docker")
-            .args(args_str.as_str().split(' '))
-            .status()?;
-        if !status.success() {
-            return Err(anyhow::anyhow!("registry setup failed"));
-        }
-        port
-    } else {
-        // turbolift-registry is running. Return for already-running registry.
-        PORT_RE
-            .captures(&utf8)
-            .unwrap()
-            .get(1)
-            .unwrap()
-            .as_str()
-            .to_string()
-    };
-
-    // return local ip + the registry port
-    let interfaces = get_if_addrs::get_if_addrs()?;
-    for interface in &interfaces {
-        if (interface.name == "en0") || (interface.name == "eth0") {
-            // todo support other network interfaces and figure out a better way to choose the interface
-            let ip = interface.addr.ip();
-            let ip_and_port = "http://".to_string() + &ip.to_string() + ":" + &port;
-            return Ok(Url::from_str(&ip_and_port)?);
-        }
-    }
-    Err(anyhow::anyhow!(
-        "no en0/eth0 interface found. interfaces: {:?}",
-        interfaces
-    ))
-}
-
 fn make_image(
     function_name: &str,
     project_tar: &[u8],
@@ -322,12 +265,17 @@ fn make_image(
     let tar_file_name = "source.tar";
     let tar_path = build_dir_canonical.join(tar_file_name);
     let docker_file = format!(
-        "FROM rustlang/rust:nightly
+        "FROM alpine:3
+RUN apk add curl libgcc gcc musl-dev make zlib-dev openssl-dev perl git
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly
+ENV PATH=/root/.cargo/bin:$PATH
+RUN rustup toolchain install nightly-2020-09-28
+RUN rustup default nightly-2020-09-28
 COPY {} {}
 RUN cat {} | tar xvf -
 WORKDIR {}
-RUN RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo build --release
-CMD RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run --release 127.0.0.1:5000",
+RUN RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo build
+CMD RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run localhost:5000",
         tar_file_name, tar_file_name, tar_file_name, function_name
     );
     std::fs::write(&dockerfile_path, docker_file)?;
@@ -390,42 +338,42 @@ CMD RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run --release 127.0.0.1:500
 impl Drop for K8s {
     fn drop(&mut self) {
         // delete the associated services and deployments from the functions we distributed
-        let rt = tokio::runtime::Runtime::new().unwrap();
-        rt.block_on(async {
-            let deployment_client = Client::try_default().compat().await.unwrap();
-            let deployments: Api<Deployment> =
-                Api::namespaced(deployment_client, TURBOLIFT_K8S_NAMESPACE);
-            let service_client = Client::try_default().compat().await.unwrap();
-            let services: Api<Service> = Api::namespaced(service_client, TURBOLIFT_K8S_NAMESPACE);
-
-            let distributed_functions = self.fn_names_to_services.keys();
-            for function in distributed_functions {
-                let service = function_to_service_name(function);
-                services
-                    .delete(&service, &Default::default())
-                    .compat()
-                    .await
-                    .unwrap();
-                let deployment = function_to_deployment_name(function);
-                deployments
-                    .delete(&deployment, &Default::default())
-                    .compat()
-                    .await
-                    .unwrap();
-            }
-        });
-
-        // delete the local registry
-        let registry_deletion_status = Command::new("docker")
-            .arg("rmi")
-            .arg("$(docker images |grep 'turbolift-registry')")
-            .status()
-            .unwrap();
-        if !registry_deletion_status.success() {
-            eprintln!(
-                "could not delete turblift registry docker image. error code: {}",
-                registry_deletion_status.code().unwrap()
-            );
-        }
+        // let rt = tokio::runtime::Runtime::new().unwrap();
+        // rt.block_on(async {
+        //     let deployment_client = Client::try_default().compat().await.unwrap();
+        //     let deployments: Api<Deployment> =
+        //         Api::namespaced(deployment_client, TURBOLIFT_K8S_NAMESPACE);
+        //     let service_client = Client::try_default().compat().await.unwrap();
+        //     let services: Api<Service> = Api::namespaced(service_client, TURBOLIFT_K8S_NAMESPACE);
+        //
+        //     let distributed_functions = self.fn_names_to_services.keys();
+        //     for function in distributed_functions {
+        //         let service = function_to_service_name(function);
+        //         services
+        //             .delete(&service, &Default::default())
+        //             .compat()
+        //             .await
+        //             .unwrap();
+        //         let deployment = function_to_deployment_name(function);
+        //         deployments
+        //             .delete(&deployment, &Default::default())
+        //             .compat()
+        //             .await
+        //             .unwrap();
+        //     }
+        // });
+        //
+        // // delete the local registry
+        // let registry_deletion_status = Command::new("docker")
+        //     .arg("rmi")
+        //     .arg("$(docker images |grep 'turbolift-registry')")
+        //     .status()
+        //     .unwrap();
+        // if !registry_deletion_status.success() {
+        //     eprintln!(
+        //         "could not delete turblift registry docker image. error code: {}",
+        //         registry_deletion_status.code().unwrap()
+        //     );
+        // }
     }
 }
diff --git a/turbolift_internals/src/utils.rs b/turbolift_internals/src/utils.rs
index e9f02cdd..52e10e44 100644
--- a/turbolift_internals/src/utils.rs
+++ b/turbolift_internals/src/utils.rs
@@ -1,11 +1,5 @@
-use std::net::UdpSocket;
-
 #[cfg(target_family = "unix")]
 pub use std::os::unix::fs::symlink as symlink_dir;
 
 #[cfg(target_family = "windows")]
 pub use std::os::windows::fs::symlink_dir;
-
-pub fn get_open_socket() -> std::io::Result<UdpSocket> {
-    UdpSocket::bind("127.0.0.1:0")
-}

From 7ddb8dff0dd4f540af5480b175a7d701e4f53955 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Mon, 2 Nov 2020 16:25:14 +0100
Subject: [PATCH 077/145] update CI to use new k8s testing scheme

---
 .github/workflows/examples.yml                | 25 +++++++++++--------
 .../run_distributed_tests.sh                  |  4 +--
 .../run_non_distributed_tests.sh              |  4 +--
 .../{run_tests_locally.sh => run_tests.sh}    |  2 +-
 4 files changed, 20 insertions(+), 15 deletions(-)
 rename examples/kubernetes_example/{run_tests_locally.sh => run_tests.sh} (89%)

diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml
index 544f3584..4d772f34 100644
--- a/.github/workflows/examples.yml
+++ b/.github/workflows/examples.yml
@@ -22,15 +22,20 @@ jobs:
           path: './turbolift'
       - name: install rustup and rust nightly
         run: |
-            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
-            rustup toolchain install nightly
-      - name: run non-distributed example without cluster env
+            curl --proto '=https' --tlsv1.2 -sSf gihttps://sh.rustup.rs | sh -s -- -y --default-toolchain nightly
+            rustup toolchain install nightly-2020-09-28
+            rustup default nightly-2020-09-28
+      - name: install kubectl
         run: |
-            cd turbolift/examples/kubernetes_example
-            sh run_non_distributed_tests.sh
-      - uses: engineerd/setup-kind@v0.4.0
-      - name: run distributed and non-distributed examples
+            curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl"
+            chmod +x ./kubectl
+            sudo mv ./kubectl /usr/local/bin/kubectl
+      - name: install kind
+        run: |
+            curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.9.0/kind-linux-amd64
+            chmod +x ./kind
+            PATH=$PATH:~/kind
+      - name: run tests
         run: |
-          cd turbolift/examples/kubernetes_example
-          sh run_non_distributed_tests.sh
-          sh run_distributed_tests.sh
+            cd turbolift/examples/kubernetes_example
+            sh run_tests.sh
diff --git a/examples/kubernetes_example/run_distributed_tests.sh b/examples/kubernetes_example/run_distributed_tests.sh
index 0e4038e6..5e9e3f12 100644
--- a/examples/kubernetes_example/run_distributed_tests.sh
+++ b/examples/kubernetes_example/run_distributed_tests.sh
@@ -4,5 +4,5 @@
 set -e
 
 # run distributed tests
-RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test --features distributed -- --nocapture
-RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run --features distributed
+RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo test --features distributed -- --nocapture
+RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run --features distributed
diff --git a/examples/kubernetes_example/run_non_distributed_tests.sh b/examples/kubernetes_example/run_non_distributed_tests.sh
index 82a7d058..dbc08531 100644
--- a/examples/kubernetes_example/run_non_distributed_tests.sh
+++ b/examples/kubernetes_example/run_non_distributed_tests.sh
@@ -4,5 +4,5 @@
 set -e
 
 # run non-distributed tests
-RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test -- --nocapture
-RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run
+RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo test -- --nocapture
+RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run
diff --git a/examples/kubernetes_example/run_tests_locally.sh b/examples/kubernetes_example/run_tests.sh
similarity index 89%
rename from examples/kubernetes_example/run_tests_locally.sh
rename to examples/kubernetes_example/run_tests.sh
index 1887ffaa..fdfcfea3 100644
--- a/examples/kubernetes_example/run_tests_locally.sh
+++ b/examples/kubernetes_example/run_tests.sh
@@ -15,7 +15,7 @@ set -e
 . ./run_non_distributed_tests.sh || kind delete
 
 # run distributed tests
-. ./run_distributed_tests.sh || kind delete
+. ./run_distributed_tests.sh || kubectl get pods
 
 # delete cluster
 kind delete

From 81989c12d55a5df666318a3b50bd83b5e3bd4aa7 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Tue, 3 Nov 2020 12:36:36 +0100
Subject: [PATCH 078/145] switch to using microk8s in test env

---
 examples/kubernetes_example/run_tests.sh  | 18 +++++-----
 examples/kubernetes_example/setup_kind.sh | 42 -----------------------
 turbolift_internals/src/kubernetes.rs     |  2 +-
 3 files changed, 11 insertions(+), 51 deletions(-)
 delete mode 100644 examples/kubernetes_example/setup_kind.sh

diff --git a/examples/kubernetes_example/run_tests.sh b/examples/kubernetes_example/run_tests.sh
index fdfcfea3..fce34deb 100644
--- a/examples/kubernetes_example/run_tests.sh
+++ b/examples/kubernetes_example/run_tests.sh
@@ -1,21 +1,23 @@
 #!/usr/bin/env sh
 
-# assumes cargo, rust nightly, kind, and kubectl are installed. run from turbolift/examples/kubernetes_example
+# assumes cargo, rust nightly, microk8s, and kubectl are installed. run from turbolift/examples/kubernetes_example
+# microk8s needs to have the local registry feature.
 
 # error if any command fails
 set -e
 
+# stop microk8s
+microk8s stop
+
 # run non-distributed tests without cluster
 . ./run_non_distributed_tests.sh
 
-# generate cluster
-. ./setup_kind.sh || kind delete
+# start cluster
+microk8s start
+microk8s status --wait-ready
 
 # re-run non-distributed tests
-. ./run_non_distributed_tests.sh || kind delete
+. ./run_non_distributed_tests.sh
 
 # run distributed tests
-. ./run_distributed_tests.sh || kubectl get pods
-
-# delete cluster
-kind delete
+. ./run_distributed_tests.sh
diff --git a/examples/kubernetes_example/setup_kind.sh b/examples/kubernetes_example/setup_kind.sh
deleted file mode 100644
index b72cc66a..00000000
--- a/examples/kubernetes_example/setup_kind.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/bin/sh
-# from https://kind.sigs.k8s.io/docs/user/local-registry/
-
-set -o errexit
-
-# create registry container unless it already exists
-reg_name='kind-registry'
-reg_port='5000'
-running="$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)"
-if [ "${running}" != 'true' ]; then
-  docker run \
-    -d --restart=always -p "${reg_port}:5000" --name "${reg_name}" \
-    registry:2
-fi
-
-# create a cluster with the local registry enabled in containerd
-cat <<EOF | kind create cluster --config=-
-kind: Cluster
-apiVersion: kind.x-k8s.io/v1alpha4
-containerdConfigPatches:
-- |-
-  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:${reg_port}"]
-    endpoint = ["http://${reg_name}:${reg_port}"]
-EOF
-
-# connect the registry to the cluster network
-# (the network may already be connected)
-docker network connect "kind" "${reg_name}" || true
-
-# Document the local registry
-# https://github.com/kubernetes/enhancements/tree/master/keps/sig-cluster-lifecycle/generic/1755-communicating-a-local-registry
-cat <<EOF | kubectl apply -f -
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: local-registry-hosting
-  namespace: kube-public
-data:
-  localRegistryHosting.v1: |
-    host: "localhost:${reg_port}"
-    help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
-EOF
diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index ee71d028..4d912bdc 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -17,7 +17,7 @@ use crate::distributed_platform::{
 };
 
 const TURBOLIFT_K8S_NAMESPACE: &str = "default";
-const LOCAL_REGISTRY_URL: &str = "http://localhost:5000";
+const LOCAL_REGISTRY_URL: &str = "http://localhost:32000";
 type ImageTag = String;
 
 /// `K8s` is the interface for turning rust functions into autoscaling microservices

From 36d460529d22a710a2cf0caaa71d0fa74790980b Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Tue, 3 Nov 2020 19:22:00 +0100
Subject: [PATCH 079/145] switch from using alpine to ubuntu

---
 turbolift_internals/src/kubernetes.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 4d912bdc..c6310aff 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -265,8 +265,8 @@ fn make_image(
     let tar_file_name = "source.tar";
     let tar_path = build_dir_canonical.join(tar_file_name);
     let docker_file = format!(
-        "FROM alpine:3
-RUN apk add curl libgcc gcc musl-dev make zlib-dev openssl-dev perl git
+        "FROM ubuntu:latest
+RUN apt-get update && apt-get install -y curl && rm -rf /var/lib/apt/lists/*
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly
 ENV PATH=/root/.cargo/bin:$PATH
 RUN rustup toolchain install nightly-2020-09-28

From 30f1593b45e423351c11a7b623edcb761f317a32 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Tue, 3 Nov 2020 20:50:38 +0100
Subject: [PATCH 080/145] consolidate rustup commands

---
 turbolift_internals/src/kubernetes.rs | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index c6310aff..722ec755 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -267,10 +267,8 @@ fn make_image(
     let docker_file = format!(
         "FROM ubuntu:latest
 RUN apt-get update && apt-get install -y curl && rm -rf /var/lib/apt/lists/*
-RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly-2020-09-28
 ENV PATH=/root/.cargo/bin:$PATH
-RUN rustup toolchain install nightly-2020-09-28
-RUN rustup default nightly-2020-09-28
 COPY {} {}
 RUN cat {} | tar xvf -
 WORKDIR {}

From 1476a67b24c9f52f794f23e22547ac9a2113521e Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Tue, 3 Nov 2020 22:36:15 +0100
Subject: [PATCH 081/145] include rust compiler deps

---
 turbolift_internals/src/kubernetes.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 722ec755..2bd52604 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -266,7 +266,7 @@ fn make_image(
     let tar_path = build_dir_canonical.join(tar_file_name);
     let docker_file = format!(
         "FROM ubuntu:latest
-RUN apt-get update && apt-get install -y curl && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install -y curl gcc libssl-dev pkg-config && rm -rf /var/lib/apt/lists/*
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly-2020-09-28
 ENV PATH=/root/.cargo/bin:$PATH
 COPY {} {}

From 476058a0316893f9ecb19fe9c175ff1f809d66b3 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Wed, 4 Nov 2020 10:47:34 +0100
Subject: [PATCH 082/145] set timezone in container to UTC

---
 turbolift_internals/src/kubernetes.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 2bd52604..f391ea9a 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -266,6 +266,8 @@ fn make_image(
     let tar_path = build_dir_canonical.join(tar_file_name);
     let docker_file = format!(
         "FROM ubuntu:latest
+ENV TZ=Etc/UTC
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 RUN apt-get update && apt-get install -y curl gcc libssl-dev pkg-config && rm -rf /var/lib/apt/lists/*
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly-2020-09-28
 ENV PATH=/root/.cargo/bin:$PATH

From 59c55edc3d3e1158cd70581923025abe72edfff2 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Wed, 4 Nov 2020 11:42:20 +0100
Subject: [PATCH 083/145] add comments to generated dockerfile

---
 turbolift_internals/src/kubernetes.rs | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index f391ea9a..531d13e4 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -266,14 +266,27 @@ fn make_image(
     let tar_path = build_dir_canonical.join(tar_file_name);
     let docker_file = format!(
         "FROM ubuntu:latest
+# set timezone (otherwise tzinfo stops dep installation with prompt for time zone)
 ENV TZ=Etc/UTC
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+# install curl and rust deps
 RUN apt-get update && apt-get install -y curl gcc libssl-dev pkg-config && rm -rf /var/lib/apt/lists/*
+
+# install rustup
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly-2020-09-28
 ENV PATH=/root/.cargo/bin:$PATH
+
+# copy tar file
 COPY {} {}
+
+# unpack tar
 RUN cat {} | tar xvf -
+
+# enter into unpacked source directory
 WORKDIR {}
+
+# build and run release
 RUN RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo build
 CMD RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run localhost:5000",
         tar_file_name, tar_file_name, tar_file_name, function_name

From 84264e9b0dbc3c7baeff18a680985b89edc0a8a1 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Wed, 4 Nov 2020 12:01:29 +0100
Subject: [PATCH 084/145] build k8s version as debug when project is built in
 debug, and release when project is built with --release

---
 turbolift_internals/src/build_project.rs |  4 ++--
 turbolift_internals/src/kubernetes.rs    | 12 +++++++++---
 turbolift_internals/src/utils.rs         | 15 +++++++++++++++
 3 files changed, 26 insertions(+), 5 deletions(-)

diff --git a/turbolift_internals/src/build_project.rs b/turbolift_internals/src/build_project.rs
index 6cb4896b..623b2392 100644
--- a/turbolift_internals/src/build_project.rs
+++ b/turbolift_internals/src/build_project.rs
@@ -4,7 +4,7 @@ use std::path::{Path, PathBuf};
 use std::process::Command;
 use std::str::FromStr;
 
-use crate::utils::symlink_dir;
+use crate::utils::{symlink_dir, RELEASE_FLAG};
 
 pub fn edit_cargo_file(
     original_project_source_dir: &Path,
@@ -105,7 +105,7 @@ pub fn lint(proj_path: &Path) -> anyhow::Result<()> {
 pub fn make_executable(proj_path: &Path, dest: Option<&Path>) -> anyhow::Result<()> {
     let status = Command::new("cargo")
         .current_dir(proj_path)
-        .args("build --release".split(' '))
+        .args(format!("build {}", RELEASE_FLAG).as_str().trim().split(' '))
         .status()?;
 
     if !status.success() {
diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 531d13e4..f0ef943d 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -15,6 +15,7 @@ use url::Url;
 use crate::distributed_platform::{
     ArgsString, DistributionPlatform, DistributionResult, JsonResponse,
 };
+use crate::utils::RELEASE_FLAG;
 
 const TURBOLIFT_K8S_NAMESPACE: &str = "default";
 const LOCAL_REGISTRY_URL: &str = "http://localhost:32000";
@@ -287,9 +288,14 @@ RUN cat {} | tar xvf -
 WORKDIR {}
 
 # build and run release
-RUN RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo build
-CMD RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run localhost:5000",
-        tar_file_name, tar_file_name, tar_file_name, function_name
+RUN RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo build {}
+CMD RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run {} localhost:5000",
+        tar_file_name,
+        tar_file_name,
+        tar_file_name,
+        function_name,
+        RELEASE_FLAG,
+        RELEASE_FLAG
     );
     std::fs::write(&dockerfile_path, docker_file)?;
     std::fs::write(&tar_path, project_tar)?;
diff --git a/turbolift_internals/src/utils.rs b/turbolift_internals/src/utils.rs
index 52e10e44..fc5211d2 100644
--- a/turbolift_internals/src/utils.rs
+++ b/turbolift_internals/src/utils.rs
@@ -3,3 +3,18 @@ pub use std::os::unix::fs::symlink as symlink_dir;
 
 #[cfg(target_family = "windows")]
 pub use std::os::windows::fs::symlink_dir;
+
+#[cfg(not(debug_assertions))]
+pub const IS_RELEASE: bool = true;
+
+#[cfg(debug_assertions)]
+pub const IS_RELEASE: bool = false;
+
+/// is --release if built with release flag, otherwise empty string
+pub const RELEASE_FLAG: &str = {
+    if IS_RELEASE {
+        "--release"
+    } else {
+        ""
+    }
+};

From c62847c9e865662895f771bbb6147b0115ea98fa Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Wed, 4 Nov 2020 13:21:27 +0100
Subject: [PATCH 085/145] attempt to fix microk8s ci

It seems it is impossible to support microk8s in github actions without doing
some virtualization, since we need to modify the docker config to allow insecure
registries to use microk8s's builtin repo. While
[private registries](https://github.community/t/github-actions-new-pulling-from-private-docker-repositories/16089/19)
are allowed in github actions, it doesn't seem that we can enable insecure
registries, per
https://github.community/t/adding-insecure-registry-in-docker-daemon/18207.
---
 .github/workflows/examples.yml           | 18 +++++++-----------
 examples/kubernetes_example/run_tests.sh |  1 -
 turbolift_internals/src/build_project.rs |  8 ++++++--
 turbolift_internals/src/kubernetes.rs    |  2 +-
 turbolift_internals/src/utils.rs         |  2 +-
 5 files changed, 15 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml
index 4d772f34..e7fde872 100644
--- a/.github/workflows/examples.yml
+++ b/.github/workflows/examples.yml
@@ -20,21 +20,17 @@ jobs:
       - uses: actions/checkout@v2
         with:
           path: './turbolift'
+      - uses: dominicburkart/microk8s-actions@1629a09e791146e6be04134768ed6ac2dc60dc83
+        with:
+          channel: '1.19/stable'
+          rbac: 'true'
+          dns: 'true'
+          storage: 'true'
       - name: install rustup and rust nightly
         run: |
-            curl --proto '=https' --tlsv1.2 -sSf gihttps://sh.rustup.rs | sh -s -- -y --default-toolchain nightly
+            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly
             rustup toolchain install nightly-2020-09-28
             rustup default nightly-2020-09-28
-      - name: install kubectl
-        run: |
-            curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl"
-            chmod +x ./kubectl
-            sudo mv ./kubectl /usr/local/bin/kubectl
-      - name: install kind
-        run: |
-            curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.9.0/kind-linux-amd64
-            chmod +x ./kind
-            PATH=$PATH:~/kind
       - name: run tests
         run: |
             cd turbolift/examples/kubernetes_example
diff --git a/examples/kubernetes_example/run_tests.sh b/examples/kubernetes_example/run_tests.sh
index fce34deb..5e16c9ba 100644
--- a/examples/kubernetes_example/run_tests.sh
+++ b/examples/kubernetes_example/run_tests.sh
@@ -14,7 +14,6 @@ microk8s stop
 
 # start cluster
 microk8s start
-microk8s status --wait-ready
 
 # re-run non-distributed tests
 . ./run_non_distributed_tests.sh
diff --git a/turbolift_internals/src/build_project.rs b/turbolift_internals/src/build_project.rs
index 623b2392..de5c1071 100644
--- a/turbolift_internals/src/build_project.rs
+++ b/turbolift_internals/src/build_project.rs
@@ -4,7 +4,7 @@ use std::path::{Path, PathBuf};
 use std::process::Command;
 use std::str::FromStr;
 
-use crate::utils::{symlink_dir, RELEASE_FLAG};
+use crate::utils::{symlink_dir, IS_RELEASE, RELEASE_FLAG};
 
 pub fn edit_cargo_file(
     original_project_source_dir: &Path,
@@ -119,7 +119,11 @@ pub fn make_executable(proj_path: &Path, dest: Option<&Path>) -> anyhow::Result<
             let cargo_path = proj_path.join("Cargo.toml");
             let parsed_toml: cargo_toml2::CargoToml = cargo_toml2::from_path(cargo_path)?;
             let project_name = parsed_toml.package.name;
-            let local_path = "target/release/".to_string() + &project_name;
+            let local_path = if IS_RELEASE {
+                "target/release/".to_string() + &project_name
+            } else {
+                "target/debug/".to_string() + &project_name
+            };
             proj_path.canonicalize().unwrap().join(&local_path)
         };
         fs::rename(&executable_path, destination)?;
diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index f0ef943d..9a90b26c 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -18,7 +18,7 @@ use crate::distributed_platform::{
 use crate::utils::RELEASE_FLAG;
 
 const TURBOLIFT_K8S_NAMESPACE: &str = "default";
-const LOCAL_REGISTRY_URL: &str = "http://localhost:32000";
+const LOCAL_REGISTRY_URL: &str = "http://127.0.0.1:32000";
 type ImageTag = String;
 
 /// `K8s` is the interface for turning rust functions into autoscaling microservices
diff --git a/turbolift_internals/src/utils.rs b/turbolift_internals/src/utils.rs
index fc5211d2..15a5ef7f 100644
--- a/turbolift_internals/src/utils.rs
+++ b/turbolift_internals/src/utils.rs
@@ -10,7 +10,7 @@ pub const IS_RELEASE: bool = true;
 #[cfg(debug_assertions)]
 pub const IS_RELEASE: bool = false;
 
-/// is --release if built with release flag, otherwise empty string
+/// is "--release" if built with release flag, otherwise empty string
 pub const RELEASE_FLAG: &str = {
     if IS_RELEASE {
         "--release"

From 23c0bc8f285c4af79e0a4aef78d4decfd4b0de31 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Fri, 20 Nov 2020 19:10:27 +0100
Subject: [PATCH 086/145] compact rust nightly install command

---
 .github/workflows/examples.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml
index e7fde872..f3a2a6ba 100644
--- a/.github/workflows/examples.yml
+++ b/.github/workflows/examples.yml
@@ -28,9 +28,7 @@ jobs:
           storage: 'true'
       - name: install rustup and rust nightly
         run: |
-            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly
-            rustup toolchain install nightly-2020-09-28
-            rustup default nightly-2020-09-28
+            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly-2020-09-28
       - name: run tests
         run: |
             cd turbolift/examples/kubernetes_example

From c48813e0cf2a35f93cca21ad89bde18cd7c96bfb Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Tue, 24 Nov 2020 18:56:29 +0100
Subject: [PATCH 087/145] add tracing

---
 Cargo.toml                                  |  2 +
 examples/kubernetes_example/Cargo.toml      |  5 ++
 examples/kubernetes_example/src/main.rs     | 13 +++-
 examples/local_queue_example/Cargo.toml     |  6 +-
 examples/local_queue_example/src/main.rs    | 14 ++++-
 src/lib.rs                                  |  1 +
 turbolift_internals/Cargo.toml              |  2 +
 turbolift_internals/src/build_project.rs    |  4 ++
 turbolift_internals/src/extract_function.rs | 15 ++++-
 turbolift_internals/src/kubernetes.rs       | 67 ++++++++++-----------
 turbolift_internals/src/local_queue.rs      | 16 +++--
 turbolift_macros/Cargo.toml                 |  2 +
 turbolift_macros/src/lib.rs                 | 27 ++++++---
 13 files changed, 117 insertions(+), 57 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 042ad1ee..3e17199f 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -18,3 +18,5 @@ chrono = { version = "0.4", optional = true }
 actix-web = { version = "3" }
 serde_json = { version = "1" }
 tokio-compat-02 = { version = "0.1" }
+tracing = {version="0.1", features=["attributes"]}
+tracing-futures = "0.2.4"
diff --git a/examples/kubernetes_example/Cargo.toml b/examples/kubernetes_example/Cargo.toml
index 2f1a178f..2f227eb9 100644
--- a/examples/kubernetes_example/Cargo.toml
+++ b/examples/kubernetes_example/Cargo.toml
@@ -14,3 +14,8 @@ lazy_static = "1"
 futures = "0.3"
 cute = "0.3"
 turbolift = { path="../../" }
+
+# for printing out tracing
+tracing = "0.1"
+tracing-futures = "0.2"
+tracing-subscriber = {version="0.2", features=["fmt"]}
diff --git a/examples/kubernetes_example/src/main.rs b/examples/kubernetes_example/src/main.rs
index fa733850..aad94941 100644
--- a/examples/kubernetes_example/src/main.rs
+++ b/examples/kubernetes_example/src/main.rs
@@ -5,6 +5,8 @@ extern crate cute;
 use futures::future::try_join_all;
 use rand::{thread_rng, Rng};
 use tokio::sync::Mutex;
+use tracing;
+use tracing_subscriber;
 
 use turbolift::kubernetes::K8s;
 use turbolift::on;
@@ -24,11 +26,20 @@ fn random_numbers() -> Vec<u64> {
 }
 
 fn main() {
+    // use tracing.rs to print info about the program to stdout
+    tracing_subscriber::fmt()
+        .with_max_level(tracing::Level::TRACE)
+        .with_writer(std::io::stdout)
+        .init();
+
     let input = random_numbers();
     let futures = c![square(*int), for int in &input];
     let mut rt = tokio::runtime::Runtime::new().unwrap();
     let output = rt.block_on(try_join_all(futures)).unwrap();
-    println!("input: {:?}\noutput: {:?}", input, output);
+    println!(
+        "\n\ncomputation complete.\ninput: {:?}\noutput: {:?}",
+        input, output
+    );
     if output != c![x*x, for x in input] {
         std::process::exit(1)
     }
diff --git a/examples/local_queue_example/Cargo.toml b/examples/local_queue_example/Cargo.toml
index 60935de1..abcd6043 100644
--- a/examples/local_queue_example/Cargo.toml
+++ b/examples/local_queue_example/Cargo.toml
@@ -15,5 +15,7 @@ lazy_static = "1"
 tokio = {version="0.3", features=["full"]}
 turbolift = { path="../../" }
 
-# these are dependencies that need to be bundled into turbolift, but debugging how to reexport proc macros is hard.
-cached = "0.19"
+# for printing out tracing
+tracing = "0.1"
+tracing-futures = "0.2"
+tracing-subscriber = {version="0.2", features=["fmt"]}
diff --git a/examples/local_queue_example/src/main.rs b/examples/local_queue_example/src/main.rs
index c277be43..6b09adc9 100644
--- a/examples/local_queue_example/src/main.rs
+++ b/examples/local_queue_example/src/main.rs
@@ -7,6 +7,9 @@ use turbolift::on;
 extern crate lazy_static;
 use tokio::sync::Mutex;
 
+use tracing::{self, info};
+use tracing_subscriber;
+
 lazy_static! {
     static ref LOCAL: Mutex<LocalQueue> = Mutex::new(LocalQueue::new());
 }
@@ -17,6 +20,12 @@ fn identity(b: bool) -> bool {
 }
 
 fn main() {
+    // use tracing.rs to print info about the program to stdout
+    tracing_subscriber::fmt()
+        .with_max_level(tracing::Level::TRACE)
+        .with_writer(std::io::stdout)
+        .init();
+
     let input = vec![rand::random(), rand::random(), rand::random()];
     let futures = {
         let mut v = Vec::new();
@@ -27,7 +36,10 @@ fn main() {
     };
     let mut rt = tokio::runtime::Runtime::new().unwrap();
     let output = rt.block_on(try_join_all(futures)).unwrap();
-    println!("input: {:?}\noutput: {:?}", input, output);
+    println!(
+        "\n\nAll responses received.\ninput: {:?}\noutput: {:?}",
+        input, output
+    );
     if output != input {
         std::process::exit(1)
     }
diff --git a/src/lib.rs b/src/lib.rs
index 35443d1b..02f4b3cb 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -4,6 +4,7 @@ pub use chrono;
 pub use actix_web;
 pub use serde_json;
 pub use tokio_compat_02;
+pub use tracing;
 
 pub use distributed_platform::{DistributionPlatform, DistributionResult};
 pub use turbolift_internals::*;
diff --git a/turbolift_internals/Cargo.toml b/turbolift_internals/Cargo.toml
index bb167896..7bd1dcff 100644
--- a/turbolift_internals/Cargo.toml
+++ b/turbolift_internals/Cargo.toml
@@ -31,6 +31,8 @@ cached = "0.19"
 async-trait = "0.1"
 get_if_addrs = "0.5.3"
 regex = "1"
+tracing = {version="0.1", features=["attributes"]}
+tracing-futures = "0.2.4"
 
 # kubernetes-specific requirements
 kube = "0.43.0"
diff --git a/turbolift_internals/src/build_project.rs b/turbolift_internals/src/build_project.rs
index de5c1071..90ed6204 100644
--- a/turbolift_internals/src/build_project.rs
+++ b/turbolift_internals/src/build_project.rs
@@ -6,6 +6,7 @@ use std::str::FromStr;
 
 use crate::utils::{symlink_dir, IS_RELEASE, RELEASE_FLAG};
 
+#[tracing::instrument]
 pub fn edit_cargo_file(
     original_project_source_dir: &Path,
     cargo_path: &Path,
@@ -82,6 +83,7 @@ pub fn edit_cargo_file(
     Ok(())
 }
 
+#[tracing::instrument]
 pub fn lint(proj_path: &Path) -> anyhow::Result<()> {
     let install_status = Command::new("rustup")
         .args("component add rustfmt".split(' '))
@@ -102,6 +104,7 @@ pub fn lint(proj_path: &Path) -> anyhow::Result<()> {
     Ok(())
 }
 
+#[tracing::instrument]
 pub fn make_executable(proj_path: &Path, dest: Option<&Path>) -> anyhow::Result<()> {
     let status = Command::new("cargo")
         .current_dir(proj_path)
@@ -131,6 +134,7 @@ pub fn make_executable(proj_path: &Path, dest: Option<&Path>) -> anyhow::Result<
     Ok(())
 }
 
+#[tracing::instrument]
 pub fn check(proj_path: &Path) -> anyhow::Result<()> {
     let status = Command::new("cargo")
         .current_dir(proj_path)
diff --git a/turbolift_internals/src/extract_function.rs b/turbolift_internals/src/extract_function.rs
index 0582e185..d2644fc3 100644
--- a/turbolift_internals/src/extract_function.rs
+++ b/turbolift_internals/src/extract_function.rs
@@ -6,10 +6,9 @@ use std::path::{Path, PathBuf};
 use std::str::FromStr;
 
 use quote::ToTokens;
-use tar::{Archive, Builder};
-
 use syn::export::TokenStream2;
 use syn::spanned::Spanned;
+use tar::{Archive, Builder};
 
 use crate::distributed_platform::DistributionResult;
 use crate::CACHE_PATH;
@@ -18,6 +17,7 @@ type TypedParams = syn::punctuated::Punctuated<syn::FnArg, syn::Token![,]>;
 type UntypedParams = syn::punctuated::Punctuated<Box<syn::Pat>, syn::Token![,]>;
 type ParamTypes = syn::punctuated::Punctuated<Box<syn::Type>, syn::Token![,]>;
 
+#[tracing::instrument]
 pub fn get_fn_item(function: TokenStream) -> syn::ItemFn {
     match syn::parse2(function).unwrap() {
         syn::Item::Fn(fn_item) => fn_item,
@@ -28,6 +28,7 @@ pub fn get_fn_item(function: TokenStream) -> syn::ItemFn {
 /// wraps any calls to the target function from within its own service with the return type as
 /// if the call was made from outside the service. This is one way to allow compilation while
 /// references to the target function are in the service codebase.
+#[tracing::instrument]
 pub fn make_dummy_function(
     function: syn::ItemFn,
     redirect_fn_name: &str,
@@ -71,6 +72,7 @@ pub fn make_dummy_function(
     }
 }
 
+#[tracing::instrument]
 pub fn to_untyped_params(typed_params: TypedParams) -> UntypedParams {
     typed_params
         .into_iter()
@@ -80,7 +82,9 @@ pub fn to_untyped_params(typed_params: TypedParams) -> UntypedParams {
         })
         .collect()
 }
+
 /// params -> {param1}/{param2}/{param3}[...]
+#[tracing::instrument]
 pub fn to_path_params(untyped_params: UntypedParams) -> String {
     let open_bracket = "{";
     let close_bracket = "}".to_string();
@@ -92,6 +96,7 @@ pub fn to_path_params(untyped_params: UntypedParams) -> String {
     path_format.join("/")
 }
 
+#[tracing::instrument]
 pub fn to_param_types(typed_params: TypedParams) -> ParamTypes {
     typed_params
         .into_iter()
@@ -102,6 +107,7 @@ pub fn to_param_types(typed_params: TypedParams) -> ParamTypes {
         .collect()
 }
 
+#[tracing::instrument]
 pub fn params_json_vec(untyped_params: UntypedParams) -> TokenStream {
     let punc: Vec<String> = untyped_params
         .into_iter()
@@ -116,6 +122,7 @@ pub fn params_json_vec(untyped_params: UntypedParams) -> TokenStream {
     TokenStream::from_str(&vec_string).unwrap()
 }
 
+#[tracing::instrument]
 pub fn get_sanitized_file(function: &TokenStream) -> TokenStream {
     let span = function.span();
     let path = span.source_file().path();
@@ -150,12 +157,14 @@ pub fn get_sanitized_file(function: &TokenStream) -> TokenStream {
     TokenStream2::from_str(&sanitized_string).unwrap()
 }
 
+#[tracing::instrument]
 pub fn unpack_path_params(untyped_params: &UntypedParams) -> TokenStream {
     let n_params = untyped_params.len();
     let params: Vec<String> = (0..n_params).map(|i| format!("path.{}", i)).collect();
     TokenStream::from_str(&params.join(", ")).unwrap()
 }
 
+#[tracing::instrument]
 pub fn make_compressed_proj_src(dir: &Path) -> Vec<u8> {
     let cursor = Cursor::new(Vec::new());
     let mut archive = Builder::new(cursor);
@@ -201,6 +210,7 @@ pub fn make_compressed_proj_src(dir: &Path) -> Vec<u8> {
     archive.into_inner().unwrap().into_inner()
 }
 
+#[tracing::instrument]
 pub fn decompress_proj_src(src: &[u8], dest: &Path) -> DistributionResult<()> {
     let cursor = Cursor::new(src.to_owned());
     let mut archive = Archive::new(cursor);
@@ -208,6 +218,7 @@ pub fn decompress_proj_src(src: &[u8], dest: &Path) -> DistributionResult<()> {
 }
 
 /// assumes input is a function, not a closure.
+#[tracing::instrument]
 pub fn get_result_type(output: &syn::ReturnType) -> TokenStream2 {
     match output {
         syn::ReturnType::Default => TokenStream2::from_str("()").unwrap(),
diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 9a90b26c..a0d68f79 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -8,7 +8,6 @@ use k8s_openapi::api::core::v1::Service;
 use kube::api::{Api, PostParams};
 use kube::Client;
 use regex::Regex;
-use syn::export::Formatter;
 use tokio_compat_02::FutureExt;
 use url::Url;
 
@@ -26,6 +25,7 @@ type ImageTag = String;
 /// device at runtime.
 ///
 /// Access to the kubernetes cluster must be inferrable from the env variables at runtime.
+#[derive(Debug)]
 pub struct K8s {
     max_scale_n: u32,
     fn_names_to_services: HashMap<String, Url>,
@@ -34,6 +34,7 @@ pub struct K8s {
 
 impl K8s {
     /// returns a K8s object that does not perform autoscaling.
+    #[tracing::instrument]
     pub fn new() -> K8s {
         K8s::with_max_replicas(1)
     }
@@ -42,6 +43,7 @@ impl K8s {
     /// is not enabled. Otherwise, autoscale is automatically activated
     /// with cluster defaults and a max number of replicas *per distributed
     /// function* of `max`. Panics if `max` < 1.
+    #[tracing::instrument]
     pub fn with_max_replicas(max: u32) -> K8s {
         if max < 1 {
             panic!("max < 1 while instantiating k8s (value: {})", max)
@@ -60,19 +62,6 @@ impl Default for K8s {
     }
 }
 
-pub struct AutoscaleError {}
-impl std::error::Error for AutoscaleError {}
-impl std::fmt::Debug for AutoscaleError {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        write!(f, "error while applying autoscale")
-    }
-}
-impl std::fmt::Display for AutoscaleError {
-    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        write!(f, "error while applying autoscale")
-    }
-}
-
 fn function_to_service_name(function_name: &str) -> String {
     function_name.to_string().replace("_", "-") + "-service"
 }
@@ -104,13 +93,13 @@ impl DistributionPlatform for K8s {
         let tag_in_reg = make_image(function_name, project_tar, &registry_url)?;
         let image_url = registry_url.join(&tag_in_reg)?.as_str().to_string();
 
-        println!("wooo");
+        tracing::info!("image made. making deployment and service names.");
         let deployment_name = function_to_deployment_name(function_name);
         let service_name = function_to_service_name(function_name);
-        println!("got service_name");
+        tracing::info!("made service_name");
         let app_name = function_to_app_name(function_name);
         let container_name = function_to_container_name(function_name);
-        println!("... app_name is fine...");
+        tracing::info!("made app_name and container_name");
 
         // make deployment
         let deployment_json = serde_json::json!({
@@ -151,14 +140,14 @@ impl DistributionPlatform for K8s {
                 }
             }
         });
-        println!("deployment_json generated, {:?}", deployment_json);
+        tracing::info!("deployment_json generated");
         let deployment = serde_json::from_value(deployment_json)?;
-        println!("made deployment: {:?}", deployment);
+        tracing::info!("deployment generated");
         deployments
             .create(&PostParams::default(), &deployment)
             .compat()
             .await?;
-        println!("created deployment");
+        tracing::info!("created deployment");
 
         // make service pointing to deployment
         let service = serde_json::from_value(serde_json::json!({
@@ -180,12 +169,12 @@ impl DistributionPlatform for K8s {
                 ]
             }
         }))?;
-        println!("made service");
+        tracing::info!("made service");
         let service = services
             .create(&PostParams::default(), &service)
             .compat()
             .await?;
-        println!("created service");
+        tracing::info!("created service");
         let node_port = service
             .spec
             .expect("no specification found for service")
@@ -195,8 +184,8 @@ impl DistributionPlatform for K8s {
             .filter_map(|port| port.node_port)
             .next()
             .expect("no node port assigned to service");
-        let service_ip = format!("http://localhost:{}", node_port);
-        println!("service_ip {}", service_ip);
+        let service_ip = format!("http://127.0.0.1:{}", node_port);
+        tracing::info!("generated service_ip");
 
         // todo make sure that the pod and service were correctly started before returning
 
@@ -209,8 +198,13 @@ impl DistributionPlatform for K8s {
             let scale_status = Command::new("kubectl")
                 .args(scale_args.as_str().split(' '))
                 .status()?;
+
             if !scale_status.success() {
-                return Err(Box::new(AutoscaleError {}));
+                return Err(anyhow::anyhow!(
+                    "autoscale error: error code: {:?}",
+                    scale_status.code()
+                )
+                .into());
                 // ^ todo attach error context from child
             }
         }
@@ -229,8 +223,8 @@ impl DistributionPlatform for K8s {
         let service_base_url = self.fn_names_to_services.get(function_name).unwrap();
         let args = "./".to_string() + function_name + "/" + &params;
         let query_url = service_base_url.join(&args)?;
-        println!("sending dispatch request to {:?}", query_url);
-        let resp = Ok(self
+        tracing::info!("sending dispatch request");
+        Ok(self
             .request_client
             .get(query_url)
             .send()
@@ -238,11 +232,10 @@ impl DistributionPlatform for K8s {
             .await?
             .text()
             .compat()
-            .await?);
-        println!("dispatch returning: {:?}", resp);
-        resp
+            .await?)
     }
 
+    #[tracing::instrument]
     fn has_declared(&self, fn_name: &str) -> bool {
         self.fn_names_to_services.contains_key(fn_name)
     }
@@ -252,12 +245,13 @@ lazy_static! {
     static ref PORT_RE: Regex = Regex::new(r"0\.0\.0\.0:(\d+)->").unwrap();
 }
 
+#[tracing::instrument]
 fn make_image(
     function_name: &str,
     project_tar: &[u8],
     registry_url: &Url,
 ) -> anyhow::Result<ImageTag> {
-    println!("making image");
+    tracing::info!("making image");
     // set up directory and dockerfile
     let build_dir = std::path::PathBuf::from(format!("{}_k8s_temp_dir", function_name));
     std::fs::create_dir_all(&build_dir)?;
@@ -322,7 +316,7 @@ CMD RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run {} localhost:5000",
             function_name
         );
 
-        println!("image tag: {}", image_tag);
+        tracing::info!("made image tag");
 
         // tag image
         let tag_args = format!("image tag {} {}", function_name, image_tag);
@@ -333,28 +327,29 @@ CMD RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run {} localhost:5000",
             return Err(anyhow::anyhow!("docker image tag failure"));
         }
 
-        println!("image tag worked: {}", tag_args);
+        tracing::info!("image tagged");
 
         // push image to local repo
         let push_status = Command::new("docker")
             .arg("push")
             .arg(image_tag.clone())
             .status()?;
-        println!("docker push command did not explode");
+        tracing::info!("docker push command did not explode");
         if !push_status.success() {
             return Err(anyhow::anyhow!("docker image push failure"));
         }
 
         Ok(image_tag)
     })();
-    println!("removing build dir");
+    tracing::info!("removing build dir");
     // always remove the build directory, even on build error
     std::fs::remove_dir_all(build_dir_canonical)?;
-    println!("returning res");
+    tracing::info!("returning result");
     result
 }
 
 impl Drop for K8s {
+    #[tracing::instrument]
     fn drop(&mut self) {
         // delete the associated services and deployments from the functions we distributed
         // let rt = tokio::runtime::Runtime::new().unwrap();
diff --git a/turbolift_internals/src/local_queue.rs b/turbolift_internals/src/local_queue.rs
index e383abce..3d08b241 100644
--- a/turbolift_internals/src/local_queue.rs
+++ b/turbolift_internals/src/local_queue.rs
@@ -19,7 +19,7 @@ use crate::CACHE_PATH;
 type AddressAndPort = Url;
 type FunctionName = String;
 
-#[derive(Default)]
+#[derive(Default, Debug)]
 pub struct LocalQueue {
     fn_name_to_address: HashMap<FunctionName, AddressAndPort>, // todo hardcoded rn
     fn_name_to_process: HashMap<FunctionName, Child>,
@@ -36,6 +36,7 @@ impl LocalQueue {
 #[async_trait]
 impl DistributionPlatform for LocalQueue {
     /// declare a function. Runs once.
+    #[tracing::instrument]
     async fn declare(&mut self, function_name: &str, project_tar: &[u8]) -> DistributionResult<()> {
         let relative_build_dir = Path::new(".")
             .join(".turbolift")
@@ -53,6 +54,7 @@ impl DistributionPlatform for LocalQueue {
     }
 
     // dispatch params to a function. Runs each time the function is called.
+    #[tracing::instrument]
     async fn dispatch(
         &mut self,
         function_name: &str,
@@ -71,13 +73,13 @@ impl DistributionPlatform for LocalQueue {
                 let server_url: AddressAndPort =
                     Url::parse(&("http://".to_string() + server_address_and_port_str))?;
                 let executable = self.fn_name_to_binary_path.get(function_name).unwrap();
-                println!("spawning");
+                tracing::info!("spawning");
                 let server_handle = Command::new(executable)
                     .arg(&server_address_and_port_str)
                     .spawn()?;
-                println!("delaying");
+                tracing::info!("delaying");
                 tokio::time::sleep(Duration::from_secs(60)).await;
-                println!("delay completed");
+                tracing::info!("delay completed");
                 // ^ sleep to make sure the server is initialized before continuing
                 self.fn_name_to_address
                     .insert(function_name.to_string(), server_url.clone());
@@ -91,7 +93,7 @@ impl DistributionPlatform for LocalQueue {
         let prefixed_params = "./".to_string() + function_name + "/" + &params;
         let query_url = address_and_port.join(&prefixed_params)?;
 
-        println!("sending dispatch request to {:?}", query_url);
+        tracing::info!("sending dispatch request");
         let resp = Ok(self
             .request_client
             .get(query_url)
@@ -101,10 +103,11 @@ impl DistributionPlatform for LocalQueue {
             .text()
             .compat()
             .await?);
-        println!("dispatch returning: {:?}", resp);
+        tracing::info!("received response");
         resp
     }
 
+    #[tracing::instrument]
     fn has_declared(&self, fn_name: &str) -> bool {
         self.fn_name_to_binary_path.contains_key(fn_name)
     }
@@ -112,6 +115,7 @@ impl DistributionPlatform for LocalQueue {
 
 impl Drop for LocalQueue {
     /// terminate all servers when program is finished
+    #[tracing::instrument]
     fn drop(&mut self) {
         self.fn_name_to_process
             .drain()
diff --git a/turbolift_macros/Cargo.toml b/turbolift_macros/Cargo.toml
index a39d1eab..f5404c5c 100644
--- a/turbolift_macros/Cargo.toml
+++ b/turbolift_macros/Cargo.toml
@@ -19,3 +19,5 @@ fs_extra = "1"
 turbolift_internals = { path="../turbolift_internals" }
 futures = "0.3"
 cached = "0.19"
+tracing = {version="0.1", features=["attributes"]}
+tracing-futures = "0.2.4"
diff --git a/turbolift_macros/src/lib.rs b/turbolift_macros/src/lib.rs
index c970c92d..d5ecb2fa 100644
--- a/turbolift_macros/src/lib.rs
+++ b/turbolift_macros/src/lib.rs
@@ -6,6 +6,7 @@ use turbolift_internals::extract_function;
 
 #[cfg(feature = "distributed")]
 #[proc_macro_attribute]
+#[tracing::instrument]
 pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenStream {
     use quote::{format_ident, ToTokens};
     use std::fs;
@@ -55,6 +56,7 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
         #target_function
 
         #[get(#wrapper_route)]
+        #[turbolift::tracing::instrument]
         async fn turbolift_wrapper(web::Path((#untyped_params_tokens)): web::Path<(#param_types)>) -> Result<HttpResponse> {
             Ok(
                 HttpResponse::Ok()
@@ -63,11 +65,13 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
         }
 
         #[actix_web::main]
+        #[turbolift::tracing::instrument]
         async fn main() -> std::io::Result<()> {
             use actix_web::{App, HttpServer};
 
             let args: Vec<String> = std::env::args().collect();
             let ip_and_port = &args[1];
+            turbolift::tracing::info!("service main() started. ip_and_port parsed.");
             HttpServer::new(
                 ||
                     App::new()
@@ -123,7 +127,10 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
 
     // lint project
     if let Err(e) = build_project::lint(&function_cache_proj_path) {
-        eprintln!("ignoring linting error: {:?}", e)
+        tracing::error!(
+            error = e.as_ref() as &(dyn std::error::Error + 'static),
+            "ignoring linting error"
+        );
     }
 
     // check project and give errors
@@ -139,7 +146,7 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
     // ^ todo
 
     // compress project source files
-    println!("on: build complete");
+    tracing::info!("build complete");
     let project_source_binary = {
         let tar = extract_function::make_compressed_proj_src(&function_cache_proj_path);
         let tar_file = CACHE_PATH.join(original_target_function_name.clone() + "_source.tar");
@@ -154,13 +161,14 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
         ))
         .expect("syntax error while embedding project tar.")
     };
-    println!("on: project_source_binary complete");
+    tracing::info!("project_source_binary complete");
 
     // generate API function for the microservice
     let declare_and_dispatch = q! {
         extern crate turbolift;
 
         // dispatch call and process response
+        #[turbolift::tracing::instrument]
         async fn #original_target_function_ident(#typed_params) ->
             turbolift::DistributionResult<#result_type> {
             use std::time::Duration;
@@ -168,24 +176,24 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
             use turbolift::DistributionResult;
             use turbolift::tokio_compat_02::FutureExt;
 
-            println!("in original target function");
+            turbolift::tracing::info!("in original target function");
 
             let mut platform = #distribution_platform.lock().await;
 
-            println!("platform generated");
+            turbolift::tracing::info!("platform generated");
 
             if !platform.has_declared(#original_target_function_name) {
-                println!("launching declare");
+                turbolift::tracing::info!("launching declare");
                 platform
                     .declare(#original_target_function_name, #project_source_binary)
                     .compat()
                     .await?;
-                println!("declare completed");
+                turbolift::tracing::info!("declare completed");
             }
 
             let params = #params_vec.join("/");
 
-            println!("launching dispatch");
+            turbolift::tracing::info!("launching dispatch");
             let resp_string = platform
                 .dispatch(
                     #original_target_function_name,
@@ -193,7 +201,7 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
                 )
                 .compat()
                 .await?;
-            println!("dispatch completed");
+            turbolift::tracing::info!("dispatch completed");
             Ok(turbolift::serde_json::from_str(&resp_string)?)
         }
     };
@@ -218,6 +226,7 @@ pub fn on(_distribution_platform: TokenStream, function_: TokenStream) -> TokenS
     let async_function = q! {
         extern crate turbolift;
 
+        #[turbolift::tracing::instrument]
         async fn #original_target_function_ident(#typed_params) -> turbolift::DistributionResult<#output_type> {
             #wrapped_original_function
             Ok(wrapped_function(#untyped_params))

From b012cb88ba8da5667b389ce8fd5f47b3ad8e1ab2 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Wed, 25 Nov 2020 17:18:26 +0100
Subject: [PATCH 088/145] don't include big binaries in traces

---
 turbolift_internals/src/extract_function.rs | 2 +-
 turbolift_internals/src/kubernetes.rs       | 4 +++-
 turbolift_internals/src/local_queue.rs      | 2 +-
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/turbolift_internals/src/extract_function.rs b/turbolift_internals/src/extract_function.rs
index d2644fc3..8b09b962 100644
--- a/turbolift_internals/src/extract_function.rs
+++ b/turbolift_internals/src/extract_function.rs
@@ -210,7 +210,7 @@ pub fn make_compressed_proj_src(dir: &Path) -> Vec<u8> {
     archive.into_inner().unwrap().into_inner()
 }
 
-#[tracing::instrument]
+#[tracing::instrument(skip(src))]
 pub fn decompress_proj_src(src: &[u8], dest: &Path) -> DistributionResult<()> {
     let cursor = Cursor::new(src.to_owned());
     let mut archive = Archive::new(cursor);
diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index a0d68f79..d0b2494c 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -80,6 +80,7 @@ fn function_to_container_name(function_name: &str) -> String {
 
 #[async_trait]
 impl DistributionPlatform for K8s {
+    #[tracing::instrument(skip(project_tar))]
     async fn declare(&mut self, function_name: &str, project_tar: &[u8]) -> DistributionResult<()> {
         // connect to cluster. tries in-cluster configuration first, then falls back to kubeconfig file.
         let deployment_client = Client::try_default().compat().await?;
@@ -214,6 +215,7 @@ impl DistributionPlatform for K8s {
         Ok(())
     }
 
+    #[tracing::instrument]
     async fn dispatch(
         &mut self,
         function_name: &str,
@@ -245,7 +247,7 @@ lazy_static! {
     static ref PORT_RE: Regex = Regex::new(r"0\.0\.0\.0:(\d+)->").unwrap();
 }
 
-#[tracing::instrument]
+#[tracing::instrument(skip(project_tar))]
 fn make_image(
     function_name: &str,
     project_tar: &[u8],
diff --git a/turbolift_internals/src/local_queue.rs b/turbolift_internals/src/local_queue.rs
index 3d08b241..6cc204aa 100644
--- a/turbolift_internals/src/local_queue.rs
+++ b/turbolift_internals/src/local_queue.rs
@@ -36,7 +36,7 @@ impl LocalQueue {
 #[async_trait]
 impl DistributionPlatform for LocalQueue {
     /// declare a function. Runs once.
-    #[tracing::instrument]
+    #[tracing::instrument(skip(project_tar))]
     async fn declare(&mut self, function_name: &str, project_tar: &[u8]) -> DistributionResult<()> {
         let relative_build_dir = Path::new(".")
             .join(".turbolift")

From 7afab27e633eb67a49f213c111b6956464d5be89 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Tue, 24 Nov 2020 20:53:55 +0100
Subject: [PATCH 089/145] debug weird file DNE error

---
 turbolift_internals/src/build_project.rs    |  2 +-
 turbolift_internals/src/extract_function.rs | 12 ++--
 turbolift_internals/src/kubernetes.rs       | 69 +++++++++++++--------
 turbolift_internals/src/utils.rs            |  4 +-
 turbolift_macros/src/lib.rs                 | 29 +++++----
 5 files changed, 69 insertions(+), 47 deletions(-)

diff --git a/turbolift_internals/src/build_project.rs b/turbolift_internals/src/build_project.rs
index 90ed6204..dc8ba51f 100644
--- a/turbolift_internals/src/build_project.rs
+++ b/turbolift_internals/src/build_project.rs
@@ -108,7 +108,7 @@ pub fn lint(proj_path: &Path) -> anyhow::Result<()> {
 pub fn make_executable(proj_path: &Path, dest: Option<&Path>) -> anyhow::Result<()> {
     let status = Command::new("cargo")
         .current_dir(proj_path)
-        .args(format!("build {}", RELEASE_FLAG).as_str().trim().split(' '))
+        .args(format!("build{}", RELEASE_FLAG).as_str().trim().split(' '))
         .status()?;
 
     if !status.success() {
diff --git a/turbolift_internals/src/extract_function.rs b/turbolift_internals/src/extract_function.rs
index 8b09b962..8bb33cd6 100644
--- a/turbolift_internals/src/extract_function.rs
+++ b/turbolift_internals/src/extract_function.rs
@@ -11,7 +11,6 @@ use syn::spanned::Spanned;
 use tar::{Archive, Builder};
 
 use crate::distributed_platform::DistributionResult;
-use crate::CACHE_PATH;
 
 type TypedParams = syn::punctuated::Punctuated<syn::FnArg, syn::Token![,]>;
 type UntypedParams = syn::punctuated::Punctuated<Box<syn::Pat>, syn::Token![,]>;
@@ -180,15 +179,18 @@ pub fn make_compressed_proj_src(dir: &Path) -> Vec<u8> {
     archive.append_dir(tar_project_base_dir, dir).unwrap();
     while !entries.is_empty() {
         let (entry_parent, entry) = entries.pop_front().unwrap();
-        if entry.file_name().to_str() == Some("target") && entry.metadata().unwrap().is_dir() {
-            // ignore target
+        if entry.metadata().unwrap().is_dir()
+            && (["target", ".git"] // todo could there be cases where removing .git messes up a dependency?
+                .contains(&entry.file_name().to_str().unwrap_or("")))
+        {
+            // ignore target and .git repository
         } else {
             let entry_path_with_parent = entry_parent.join(entry.file_name());
             if entry.path().is_dir() {
                 // ^ bug: the metadata on symlinks sometimes say that they are not directories,
                 // so we have to metadata.is_dir() || (metadata.file_type().is_symlink() && entry.path().is_dir() )
-                if CACHE_PATH.file_name().unwrap() == entry.file_name() {
-                    // don't include the cache
+                if Some("target") == entry.file_name().to_str() {
+                    // don't include any target files
                 } else {
                     archive
                         .append_dir(&entry_path_with_parent, entry.path())
diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index d0b2494c..f3c40824 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -15,9 +15,10 @@ use crate::distributed_platform::{
     ArgsString, DistributionPlatform, DistributionResult, JsonResponse,
 };
 use crate::utils::RELEASE_FLAG;
+use crate::CACHE_PATH;
 
 const TURBOLIFT_K8S_NAMESPACE: &str = "default";
-const LOCAL_REGISTRY_URL: &str = "http://127.0.0.1:32000";
+const LOCAL_REGISTRY_URL: &str = "http://localhost:32000";
 type ImageTag = String;
 
 /// `K8s` is the interface for turning rust functions into autoscaling microservices
@@ -176,6 +177,17 @@ impl DistributionPlatform for K8s {
             .compat()
             .await?;
         tracing::info!("created service");
+        let node_ip = {
+            // let stdout = Command::new("kubectl")
+            // .args("get nodes --selector=kubernetes.io/role!=master -o jsonpath={.items[*].status.addresses[?\\(@.type==\\\"InternalIP\\\"\\)].address}".split(' '))
+            // .output()
+            // .expect("error finding node ip")
+            // .stdout;
+            // String::from_utf8(stdout).expect("could not parse local node ip")
+            "192.169.0.100".to_string()
+        };
+        tracing::info!(node_ip = node_ip.as_str(), "found node ip");
+
         let node_port = service
             .spec
             .expect("no specification found for service")
@@ -185,30 +197,31 @@ impl DistributionPlatform for K8s {
             .filter_map(|port| port.node_port)
             .next()
             .expect("no node port assigned to service");
-        let service_ip = format!("http://127.0.0.1:{}", node_port);
-        tracing::info!("generated service_ip");
+        let service_ip = format!("http://{}:{}", node_ip, node_port);
+        tracing::info!(ip = service_ip.as_str(), "generated service_ip");
 
         // todo make sure that the pod and service were correctly started before returning
 
-        if self.max_scale_n > 1 {
-            // set autoscale
-            let scale_args = format!(
-                "autoscale deployment {} --max={}",
-                deployment_name, self.max_scale_n
-            );
-            let scale_status = Command::new("kubectl")
-                .args(scale_args.as_str().split(' '))
-                .status()?;
-
-            if !scale_status.success() {
-                return Err(anyhow::anyhow!(
-                    "autoscale error: error code: {:?}",
-                    scale_status.code()
-                )
-                .into());
-                // ^ todo attach error context from child
-            }
-        }
+        // if self.max_scale_n > 1 {
+        //     // set autoscale
+        //     let scale_args = format!(
+        //         "autoscale deployment {} --max={}",
+        //         deployment_name, self.max_scale_n
+        //     );
+        //     let scale_status = Command::new("kubectl")
+        //         .args(scale_args.as_str().split(' '))
+        //         .status()?;
+        //
+        //     if !scale_status.success() {
+        //         return Err(anyhow::anyhow!(
+        //             "autoscale error: error code: {:?}",
+        //             scale_status.code()
+        //         )
+        //         .into());
+        //         // ^ todo attach error context from child
+        //     }
+        // }
+
         self.fn_names_to_services
             .insert(function_name.to_string(), Url::from_str(&service_ip)?);
         // todo handle deleting the relevant service and deployment for each distributed function.
@@ -225,7 +238,7 @@ impl DistributionPlatform for K8s {
         let service_base_url = self.fn_names_to_services.get(function_name).unwrap();
         let args = "./".to_string() + function_name + "/" + &params;
         let query_url = service_base_url.join(&args)?;
-        tracing::info!("sending dispatch request");
+        tracing::info!(url = query_url.as_str(), "sending dispatch request");
         Ok(self
             .request_client
             .get(query_url)
@@ -255,7 +268,7 @@ fn make_image(
 ) -> anyhow::Result<ImageTag> {
     tracing::info!("making image");
     // set up directory and dockerfile
-    let build_dir = std::path::PathBuf::from(format!("{}_k8s_temp_dir", function_name));
+    let build_dir = CACHE_PATH.join(format!("{}_k8s_temp_dir", function_name).as_str());
     std::fs::create_dir_all(&build_dir)?;
     let build_dir_canonical = build_dir.canonicalize()?;
     let dockerfile_path = build_dir_canonical.join("Dockerfile");
@@ -283,9 +296,11 @@ RUN cat {} | tar xvf -
 # enter into unpacked source directory
 WORKDIR {}
 
-# build and run release
-RUN RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo build {}
-CMD RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run {} localhost:5000",
+# build and run project
+RUN RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo build{}
+RUN ls -latr .
+RUN ls -latr target/debug
+CMD RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run{} localhost:5000",
         tar_file_name,
         tar_file_name,
         tar_file_name,
diff --git a/turbolift_internals/src/utils.rs b/turbolift_internals/src/utils.rs
index 15a5ef7f..b288ebef 100644
--- a/turbolift_internals/src/utils.rs
+++ b/turbolift_internals/src/utils.rs
@@ -10,10 +10,10 @@ pub const IS_RELEASE: bool = true;
 #[cfg(debug_assertions)]
 pub const IS_RELEASE: bool = false;
 
-/// is "--release" if built with release flag, otherwise empty string
+/// is " --release" if built with release flag, otherwise empty string
 pub const RELEASE_FLAG: &str = {
     if IS_RELEASE {
-        "--release"
+        " --release"
     } else {
         ""
     }
diff --git a/turbolift_macros/src/lib.rs b/turbolift_macros/src/lib.rs
index d5ecb2fa..28f2bc8e 100644
--- a/turbolift_macros/src/lib.rs
+++ b/turbolift_macros/src/lib.rs
@@ -133,24 +133,29 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
         );
     }
 
-    // check project and give errors
-    build_project::check(&function_cache_proj_path).expect("error checking function");
+    // // check project and give errors
+    // build_project::check(&function_cache_proj_path).expect("error checking function");
 
+    // println!("building microservice");
     // // build project so that the deps are packaged, and if the worker has the same architecture,
-    // // they can directly use the compiled version without having to recompile. todo commented
-    // // out because the build artifacts are too large.
-    // build_project::make_executable(
-    //     &function_cache_proj_path,
-    //     None
-    // ).expect("error building function");
-    // ^ todo
+    // // they can directly use the compiled version without having to recompile. todo the build artifacts are too large.
+    // build_project::make_executable(&function_cache_proj_path, None)
+    //     .expect("error building function");
+    // // ^ todo
 
     // compress project source files
-    tracing::info!("build complete");
     let project_source_binary = {
         let tar = extract_function::make_compressed_proj_src(&function_cache_proj_path);
         let tar_file = CACHE_PATH.join(original_target_function_name.clone() + "_source.tar");
         fs::write(&tar_file, tar).expect("failure writing bin");
+        println!(
+            "tar file location: {}",
+            tar_file
+                .canonicalize()
+                .expect("error canonicalizing tar file location")
+                .to_str()
+                .unwrap()
+        );
         TokenStream2::from_str(&format!(
             "std::include_bytes!(\"{}\")",
             tar_file
@@ -161,7 +166,7 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
         ))
         .expect("syntax error while embedding project tar.")
     };
-    tracing::info!("project_source_binary complete");
+    println!("project_source_binary complete");
 
     // generate API function for the microservice
     let declare_and_dispatch = q! {
@@ -180,7 +185,7 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
 
             let mut platform = #distribution_platform.lock().await;
 
-            turbolift::tracing::info!("platform generated");
+            turbolift::tracing::info!("platform acquired");
 
             if !platform.has_declared(#original_target_function_name) {
                 turbolift::tracing::info!("launching declare");

From c69638d01084040f171e90cfa1bf0dcef4ef70e5 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Mon, 30 Nov 2020 10:50:10 +0100
Subject: [PATCH 090/145] copy instead of symlinking a local dependency that is
 also an ancestor of the project

---
 turbolift_internals/src/build_project.rs | 77 +++++++++++++++++++++---
 turbolift_macros/src/lib.rs              |  7 ++-
 2 files changed, 72 insertions(+), 12 deletions(-)

diff --git a/turbolift_internals/src/build_project.rs b/turbolift_internals/src/build_project.rs
index dc8ba51f..e56dc0aa 100644
--- a/turbolift_internals/src/build_project.rs
+++ b/turbolift_internals/src/build_project.rs
@@ -1,4 +1,5 @@
-use std::collections::HashSet;
+use std::collections::{HashSet, VecDeque};
+use std::ffi::OsStr;
 use std::fs;
 use std::path::{Path, PathBuf};
 use std::process::Command;
@@ -12,6 +13,8 @@ pub fn edit_cargo_file(
     cargo_path: &Path,
     function_name: &str,
 ) -> anyhow::Result<()> {
+    let project_canonical = original_project_source_dir.canonicalize()?;
+
     let local_deps_dir_name = ".local_deps";
     let mut parsed_toml: cargo_toml2::CargoToml = cargo_toml2::from_path(cargo_path)
         .unwrap_or_else(|_| panic!("toml at {:?} could not be read", cargo_path));
@@ -34,6 +37,7 @@ pub fn edit_cargo_file(
             cargo_toml2::Dependency::Simple(_) => None,
             cargo_toml2::Dependency::Full(detail) => Some((name, detail)),
         });
+
     let mut completed_locations = HashSet::new();
     for (name, detail) in details {
         // only descriptions with a path
@@ -42,6 +46,11 @@ pub fn edit_cargo_file(
             let canonical = original_project_source_dir.join(&buf).canonicalize()?;
             let dep_location = local_deps_cache.join(name);
 
+            // check if the dependency is an ancestor of the project
+            let is_ancestor = project_canonical
+                .ancestors()
+                .any(|p| p == canonical.as_path());
+
             // check that we don't have a naming error
             // todo: automatically handle naming conflicts by mangling the dep for one
             if completed_locations.contains(&dep_location) {
@@ -51,16 +60,31 @@ pub fn edit_cargo_file(
             }
 
             if dep_location.exists() {
-                // dependency already exists, does it point to the correct place?
-                if canonical == dep_location.canonicalize()? {
-                    // output already points to the right place, do nothing
-                } else {
-                    // output points somewhere else; delete it; if it's non-empty, error
-                    fs::remove_dir(&dep_location).unwrap();
-                    symlink_dir(&canonical, &dep_location)?;
+                if dep_canonical == dep_location.canonicalize()? {
+                    // output already points to the right place, presumably because a previous
+                    // turbolift build already created a symlink. No need to alter the
+                    // dependency cache, just point to the cache in the manifest and move on.
+                    *buf = PathBuf::from_str(".")?.join(local_deps_dir_name).join(name);
+                    continue;
                 }
+
+                // the dependency cache is not correct. We should delete what's currently there
+                // (note: symlinks will be removed, but the original files they link to will
+                // not be altered).
+                fs::remove_dir_all(&dep_location)?;
+            }
+
+            if !is_ancestor {
+                symlink_dir(&dep_canonical, &dep_location)?;
             } else {
-                symlink_dir(&canonical, &dep_location)?;
+                // copy instead of symlinking here to avoid a symlink loop that will confuse and
+                // break  the tar packer / unpacker.
+                exclusive_recursive_copy(
+                    canonical.as_path(),
+                    dep_location.as_path(),
+                    vec![project_canonical.clone()].into_iter().collect(),
+                    vec![OsStr::new("target")].into_iter().collect(),
+                )?;
             }
 
             *buf = PathBuf::from_str(".")?.join(local_deps_dir_name).join(name);
@@ -83,6 +107,41 @@ pub fn edit_cargo_file(
     Ok(())
 }
 
+/// recursively copy a directory to a target, excluding a path and its
+/// children if it exists as a descendant of the source_dir.
+fn exclusive_recursive_copy(
+    source_dir: &Path,
+    target_dir: &Path,
+    exclude_paths: HashSet<PathBuf>,
+    exclude_file_names: HashSet<&OsStr>,
+) -> anyhow::Result<()> {
+    fs::create_dir_all(target_dir)?;
+    let source_dir_canonical = source_dir.to_path_buf().canonicalize()?;
+    let mut to_check = fs::read_dir(source_dir_canonical.as_path())?.collect::<VecDeque<_>>();
+    while !to_check.is_empty() {
+        let entry = to_check.pop_front().unwrap()?;
+        let entry_path = entry.path();
+        if exclude_paths.contains(&entry_path)
+            || entry_path
+                .file_name()
+                .map_or(false, |f| exclude_file_names.contains(f))
+        {
+            // skip the excluded path (and, if it has any, all of its children)
+        } else {
+            let relative_entry_path = entry_path.strip_prefix(source_dir_canonical.as_path())?;
+            let output = target_dir.join(relative_entry_path);
+
+            if entry.metadata()?.is_dir() {
+                to_check.append(&mut fs::read_dir(entry.path())?.collect::<VecDeque<_>>());
+                fs::create_dir_all(output)?;
+            } else {
+                fs::copy(entry_path, output)?;
+            }
+        }
+    }
+    Ok(())
+}
+
 #[tracing::instrument]
 pub fn lint(proj_path: &Path) -> anyhow::Result<()> {
     let install_status = Command::new("rustup")
diff --git a/turbolift_macros/src/lib.rs b/turbolift_macros/src/lib.rs
index 28f2bc8e..ca798342 100644
--- a/turbolift_macros/src/lib.rs
+++ b/turbolift_macros/src/lib.rs
@@ -94,9 +94,10 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
         .map(|res| res.expect("could not read entry").path())
         .filter(|path| path.file_name() != CACHE_PATH.file_name())
         .filter(
-            |path| path.to_str() != Some("./target"), // todo we could shorten compile time by sharing deps in ./target,
-                                                      // but I didn't have the bandwidth to debug permissions errors caused
-                                                      // by copying all of the compiled lib files.
+            |path| path.to_str() != Some("./target"),
+            // todo we could shorten compile time by sharing deps in ./target,
+            // but I didn't have the bandwidth to debug permissions errors caused
+            // by copying all of the compiled lib files.
         )
         .collect();
     fs_extra::copy_items(

From 03a7a485b9c4bb236ad87e33694929444cf5d664 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Mon, 30 Nov 2020 15:18:14 +0100
Subject: [PATCH 091/145] fix service not exposing correctly

---
 turbolift_internals/src/build_project.rs |  6 +-
 turbolift_internals/src/kubernetes.rs    | 74 ++++++++++--------------
 2 files changed, 32 insertions(+), 48 deletions(-)

diff --git a/turbolift_internals/src/build_project.rs b/turbolift_internals/src/build_project.rs
index e56dc0aa..bafad3b1 100644
--- a/turbolift_internals/src/build_project.rs
+++ b/turbolift_internals/src/build_project.rs
@@ -43,13 +43,13 @@ pub fn edit_cargo_file(
         // only descriptions with a path
         if let Some(ref mut buf) = detail.path {
             // determine what the symlink for this dependency should be
-            let canonical = original_project_source_dir.join(&buf).canonicalize()?;
+            let dep_canonical = original_project_source_dir.join(&buf).canonicalize()?;
             let dep_location = local_deps_cache.join(name);
 
             // check if the dependency is an ancestor of the project
             let is_ancestor = project_canonical
                 .ancestors()
-                .any(|p| p == canonical.as_path());
+                .any(|p| p == dep_canonical.as_path());
 
             // check that we don't have a naming error
             // todo: automatically handle naming conflicts by mangling the dep for one
@@ -80,7 +80,7 @@ pub fn edit_cargo_file(
                 // copy instead of symlinking here to avoid a symlink loop that will confuse and
                 // break  the tar packer / unpacker.
                 exclusive_recursive_copy(
-                    canonical.as_path(),
+                    dep_canonical.as_path(),
                     dep_location.as_path(),
                     vec![project_canonical.clone()].into_iter().collect(),
                     vec![OsStr::new("target")].into_iter().collect(),
diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index f3c40824..0ee005a2 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -63,22 +63,10 @@ impl Default for K8s {
     }
 }
 
-fn function_to_service_name(function_name: &str) -> String {
-    function_name.to_string().replace("_", "-") + "-service"
-}
-
-fn function_to_deployment_name(function_name: &str) -> String {
-    function_name.to_string().replace("_", "-") + "-deployment"
-}
-
 fn function_to_app_name(function_name: &str) -> String {
     function_name.to_string().replace("_", "-")
 }
 
-fn function_to_container_name(function_name: &str) -> String {
-    function_name.to_string().replace("_", "-") + "-container"
-}
-
 #[async_trait]
 impl DistributionPlatform for K8s {
     #[tracing::instrument(skip(project_tar))]
@@ -96,40 +84,34 @@ impl DistributionPlatform for K8s {
         let image_url = registry_url.join(&tag_in_reg)?.as_str().to_string();
 
         tracing::info!("image made. making deployment and service names.");
-        let deployment_name = function_to_deployment_name(function_name);
-        let service_name = function_to_service_name(function_name);
         tracing::info!("made service_name");
         let app_name = function_to_app_name(function_name);
-        let container_name = function_to_container_name(function_name);
         tracing::info!("made app_name and container_name");
 
         // make deployment
         let deployment_json = serde_json::json!({
             "apiVersion": "apps/v1",
-            "kind": "Deployment",
             "metadata": {
-                "name": deployment_name,
-                "labels": {
-                    "app": app_name
-                }
+                "name": app_name
             },
+            "kind": "Deployment",
             "spec": {
-                "replicas": 1,
                 "selector": {
                     "matchLabels": {
-                        "app": app_name
+                        "run": app_name
                     }
                 },
+                "replicas": 1,
                 "template": {
                     "metadata": {
                         "labels": {
-                            "app": app_name
+                            "run": app_name
                         }
                     },
                     "spec": {
                         "containers": [
                             {
-                                "name": container_name,
+                                "name": app_name,
                                 "image": image_url,
                                 "ports": [
                                     {
@@ -156,23 +138,26 @@ impl DistributionPlatform for K8s {
             "apiVersion": "v1",
             "kind": "Service",
             "metadata": {
-                "name": service_name
+                "name": app_name,
+                "labels": {
+                    "run": app_name
+                }
             },
             "spec": {
-                "type": "NodePort",
-                "selector": {
-                    "app": deployment_name
-                },
                 "ports": [
                     {
                         "protocol": "TCP",
-                        "port": 5000
+                        "port": 5000,
+                        "name": "http"
                     }
-                ]
+                ],
+                "selector": {
+                    "run": app_name
+                }
             }
         }))?;
         tracing::info!("made service");
-        let service = services
+        let _service = services
             .create(&PostParams::default(), &service)
             .compat()
             .await?;
@@ -184,19 +169,20 @@ impl DistributionPlatform for K8s {
             // .expect("error finding node ip")
             // .stdout;
             // String::from_utf8(stdout).expect("could not parse local node ip")
-            "192.169.0.100".to_string()
+            "192.168.0.100".to_string()
         };
         tracing::info!(node_ip = node_ip.as_str(), "found node ip");
 
-        let node_port = service
-            .spec
-            .expect("no specification found for service")
-            .ports
-            .expect("no ports found for service")
-            .iter()
-            .filter_map(|port| port.node_port)
-            .next()
-            .expect("no node port assigned to service");
+        let node_port = 5000;
+        // let node_port = service
+        //     .spec
+        //     .expect("no specification found for service")
+        //     .ports
+        //     .expect("no ports found for service")
+        //     .iter()
+        //     .filter_map(|port| port.node_port)
+        //     .next()
+        //     .expect("no node port assigned to service");
         let service_ip = format!("http://{}:{}", node_ip, node_port);
         tracing::info!(ip = service_ip.as_str(), "generated service_ip");
 
@@ -298,9 +284,7 @@ WORKDIR {}
 
 # build and run project
 RUN RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo build{}
-RUN ls -latr .
-RUN ls -latr target/debug
-CMD RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run{} localhost:5000",
+CMD RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run{} 127.0.0.1:5000",
         tar_file_name,
         tar_file_name,
         tar_file_name,

From d59460129f4d2046319b0311157dc13ea54064ff Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Thu, 25 Feb 2021 23:22:38 +0100
Subject: [PATCH 092/145] wip: use ingress

---
 .github/workflows/examples.yml                |   7 +-
 .../run_non_distributed_tests.sh              |   2 +-
 examples/kubernetes_example/run_tests.sh      |  57 +++++-
 turbolift_internals/src/kubernetes.rs         | 190 +++++++++++-------
 turbolift_macros/src/lib.rs                   |  30 ++-
 5 files changed, 188 insertions(+), 98 deletions(-)

diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml
index f3a2a6ba..cb235713 100644
--- a/.github/workflows/examples.yml
+++ b/.github/workflows/examples.yml
@@ -20,12 +20,7 @@ jobs:
       - uses: actions/checkout@v2
         with:
           path: './turbolift'
-      - uses: dominicburkart/microk8s-actions@1629a09e791146e6be04134768ed6ac2dc60dc83
-        with:
-          channel: '1.19/stable'
-          rbac: 'true'
-          dns: 'true'
-          storage: 'true'
+      - uses: engineerd/setup-kind@v0.5.0
       - name: install rustup and rust nightly
         run: |
             curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly-2020-09-28
diff --git a/examples/kubernetes_example/run_non_distributed_tests.sh b/examples/kubernetes_example/run_non_distributed_tests.sh
index dbc08531..3b4d7c2f 100644
--- a/examples/kubernetes_example/run_non_distributed_tests.sh
+++ b/examples/kubernetes_example/run_non_distributed_tests.sh
@@ -5,4 +5,4 @@ set -e
 
 # run non-distributed tests
 RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo test -- --nocapture
-RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run
+RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run
\ No newline at end of file
diff --git a/examples/kubernetes_example/run_tests.sh b/examples/kubernetes_example/run_tests.sh
index 5e16c9ba..51692617 100644
--- a/examples/kubernetes_example/run_tests.sh
+++ b/examples/kubernetes_example/run_tests.sh
@@ -1,22 +1,59 @@
 #!/usr/bin/env sh
 
-# assumes cargo, rust nightly, microk8s, and kubectl are installed. run from turbolift/examples/kubernetes_example
-# microk8s needs to have the local registry feature.
+# assumes cargo, rust nightly, kind, and kubectl are installed. run from turbolift/examples/kubernetes_example
 
 # error if any command fails
 set -e
 
-# stop microk8s
-microk8s stop
+echo "🚡 running turbolift tests..."
 
-# run non-distributed tests without cluster
+printf "\n😤 deleting current cluster if it exists\n"
+kind delete cluster
+
+printf "\n📍 running non-distributed tests\n"
 . ./run_non_distributed_tests.sh
+echo "non-distributed tests completed."
 
-# start cluster
-microk8s start
+printf "\n👷 setting up cluster with custom ingress-compatible config\n"
+cat <<EOF | kind create cluster --config=-
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+  kubeadmConfigPatches:
+  - |
+    kind: InitConfiguration
+    nodeRegistration:
+      kubeletExtraArgs:
+        node-labels: "ingress-ready=true"
+  extraPortMappings:
+  - containerPort: 80
+    hostPort: 80
+    protocol: TCP
+  - containerPort: 443
+    hostPort: 443
+    protocol: TCP
+EOF
+kubectl cluster-info --context kind-kind
+echo "cluster initialized."
 
-# re-run non-distributed tests
-. ./run_non_distributed_tests.sh
+printf "\n🚪 adding ingress\n"
+kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/static/provider/kind/deploy.yaml
+sleep 90s # give k8s time to generate the pod ^
+printf "\n⏱️  waiting for ingress controller to be ready...\n"
+kubectl wait --namespace ingress-nginx \
+  --for=condition=ready pod \
+  --selector=app.kubernetes.io/component=controller \
+  --timeout=30m 
+
+echo "🚪 ingress ready."
 
-# run distributed tests
+printf "\n🤸‍ run distributed tests\n"
 . ./run_distributed_tests.sh
+echo "🤸 distributed tests completed."
+
+printf "\n📍 re-run non-distributed tests\n"
+. ./run_non_distributed_tests.sh
+echo "📍 non-distributed tests completed."
+
+printf "\n🚡turbolift tests complete.\n"
diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 0ee005a2..f968ec5b 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -1,26 +1,30 @@
 use std::collections::HashMap;
-use std::process::Command;
+use std::process::{Command, Stdio};
 use std::str::FromStr;
 
 use async_trait::async_trait;
 use k8s_openapi::api::apps::v1::Deployment;
-use k8s_openapi::api::core::v1::Service;
+// use k8s_openapi::api::core::v1::Service;
 use kube::api::{Api, PostParams};
 use kube::Client;
 use regex::Regex;
 use tokio_compat_02::FutureExt;
 use url::Url;
+use tokio::time::{sleep, Duration};
 
 use crate::distributed_platform::{
     ArgsString, DistributionPlatform, DistributionResult, JsonResponse,
 };
 use crate::utils::RELEASE_FLAG;
 use crate::CACHE_PATH;
+use std::io::Write;
 
 const TURBOLIFT_K8S_NAMESPACE: &str = "default";
-const LOCAL_REGISTRY_URL: &str = "http://localhost:32000";
 type ImageTag = String;
 
+pub const INTERNAL_PORT: i32 = 32766;
+pub const HOST_PORT: i32 = 80;
+
 /// `K8s` is the interface for turning rust functions into autoscaling microservices
 /// using turbolift. It requires docker and kubernetes / kubectl to already be setup on the
 /// device at runtime.
@@ -75,26 +79,24 @@ impl DistributionPlatform for K8s {
         let deployment_client = Client::try_default().compat().await?;
         let deployments: Api<Deployment> =
             Api::namespaced(deployment_client, TURBOLIFT_K8S_NAMESPACE);
-        let service_client = Client::try_default().compat().await?;
-        let services: Api<Service> = Api::namespaced(service_client, TURBOLIFT_K8S_NAMESPACE);
+        // let service_client = Client::try_default().compat().await?;
+        // let services: Api<Service> = Api::namespaced(service_client, TURBOLIFT_K8S_NAMESPACE);
 
-        // generate image & host it on a local registry
-        let registry_url = Url::parse(LOCAL_REGISTRY_URL)?;
-        let tag_in_reg = make_image(function_name, project_tar, &registry_url)?;
-        let image_url = registry_url.join(&tag_in_reg)?.as_str().to_string();
+        // generate image & push
+        let tag_in_reg = make_image(function_name, project_tar)?;
 
-        tracing::info!("image made. making deployment and service names.");
-        tracing::info!("made service_name");
+        println!("image made. making deployment and service names.");
+        println!("made service_name");
         let app_name = function_to_app_name(function_name);
-        tracing::info!("made app_name and container_name");
+        println!("made app_name and container_name");
 
         // make deployment
         let deployment_json = serde_json::json!({
             "apiVersion": "apps/v1",
+            "kind": "Deployment",
             "metadata": {
-                "name": app_name
+                "name": app_name,
             },
-            "kind": "Deployment",
             "spec": {
                 "selector": {
                     "matchLabels": {
@@ -112,10 +114,11 @@ impl DistributionPlatform for K8s {
                         "containers": [
                             {
                                 "name": app_name,
-                                "image": image_url,
+                                "image": tag_in_reg,
+                                "imagePullPolicy": "IfNotPresent", // prefer local image
                                 "ports": [
                                     {
-                                        "containerPort": 5000
+                                        "containerPort": INTERNAL_PORT
                                     }
                                 ]
                             },
@@ -124,44 +127,78 @@ impl DistributionPlatform for K8s {
                 }
             }
         });
-        tracing::info!("deployment_json generated");
+        println!("deployment_json generated");
         let deployment = serde_json::from_value(deployment_json)?;
-        tracing::info!("deployment generated");
+        println!("deployment generated");
         deployments
             .create(&PostParams::default(), &deployment)
             .compat()
             .await?;
-        tracing::info!("created deployment");
+        println!("created deployment");
 
         // make service pointing to deployment
-        let service = serde_json::from_value(serde_json::json!({
-            "apiVersion": "v1",
-            "kind": "Service",
+       if !Command::new("kubectl")
+           .args(format!("expose deployment/{}", app_name).as_str().split(' '))
+           .status()?
+           .success() {
+           tracing::error!("could not make service");
+           panic!("could not make service to expose deployment");
+       }
+
+        tracing::info!("created service");
+
+        // make ingress pointing to service
+        let ingress = serde_json::json!({
+            "apiVersion": "networking.k8s.io/v1",
+            "kind": "Ingress",
             "metadata": {
                 "name": app_name,
-                "labels": {
-                    "run": app_name
-                }
             },
             "spec": {
-                "ports": [
+                // "defaultBackend": {
+                //     "service": {
+                //         "name": app_name,
+                //         "port": {
+                //             "number": PORT
+                //         }
+                //     }
+                // }
+                "rules": [
                     {
-                        "protocol": "TCP",
-                        "port": 5000,
-                        "name": "http"
+                        "http": {
+                            "paths": [
+                                {
+                                    "path": format!("/{}", app_name),
+                                    "pathType": "Prefix",
+                                    "backend": {
+                                        "service": {
+                                            "name": app_name,
+                                            "port": {
+                                                "number": INTERNAL_PORT
+                                            }
+                                        }
+                                    }
+                                }
+                            ]
+                        }
                     }
-                ],
-                "selector": {
-                    "run": app_name
-                }
+                ]
             }
-        }))?;
-        tracing::info!("made service");
-        let _service = services
-            .create(&PostParams::default(), &service)
-            .compat()
-            .await?;
-        tracing::info!("created service");
+        });
+
+        let mut apply_ingress_child = Command::new("kubectl")
+            .args("apply -f -".split(' '))
+            .stdin(Stdio::piped())
+            .spawn()?;
+        apply_ingress_child
+            .stdin
+            .as_mut()
+            .expect("not able to write to ingress apply stdin")
+            .write_all(ingress.to_string().as_bytes())?;
+        if !apply_ingress_child.wait()?.success() {
+            panic!("failed to apply ingress: {}\nis ingress enabled on this cluster?", ingress.to_string())
+        }
+
         let node_ip = {
             // let stdout = Command::new("kubectl")
             // .args("get nodes --selector=kubernetes.io/role!=master -o jsonpath={.items[*].status.addresses[?\\(@.type==\\\"InternalIP\\\"\\)].address}".split(' '))
@@ -169,11 +206,11 @@ impl DistributionPlatform for K8s {
             // .expect("error finding node ip")
             // .stdout;
             // String::from_utf8(stdout).expect("could not parse local node ip")
-            "192.168.0.100".to_string()
+            "localhost".to_string()
         };
         tracing::info!(node_ip = node_ip.as_str(), "found node ip");
 
-        let node_port = 5000;
+        // let node_port: i32 = 5000;
         // let node_port = service
         //     .spec
         //     .expect("no specification found for service")
@@ -183,7 +220,8 @@ impl DistributionPlatform for K8s {
         //     .filter_map(|port| port.node_port)
         //     .next()
         //     .expect("no node port assigned to service");
-        let service_ip = format!("http://{}:{}", node_ip, node_port);
+        // let service_ip = format!("http://{}", node_ip, node_port);
+        let service_ip = format!("http://localhost:{}/{}/", HOST_PORT, app_name);
         tracing::info!(ip = service_ip.as_str(), "generated service_ip");
 
         // todo make sure that the pod and service were correctly started before returning
@@ -208,6 +246,9 @@ impl DistributionPlatform for K8s {
         //     }
         // }
 
+        sleep(Duration::from_secs(600)).await;
+        // todo implement the check on whether the service is running / pod failed
+
         self.fn_names_to_services
             .insert(function_name.to_string(), Url::from_str(&service_ip)?);
         // todo handle deleting the relevant service and deployment for each distributed function.
@@ -250,8 +291,11 @@ lazy_static! {
 fn make_image(
     function_name: &str,
     project_tar: &[u8],
-    registry_url: &Url,
 ) -> anyhow::Result<ImageTag> {
+    // todo: we should add some random stuff to the function_name to avoid collisions and figure
+    // out when to overwrite vs not.
+
+
     tracing::info!("making image");
     // set up directory and dockerfile
     let build_dir = CACHE_PATH.join(format!("{}_k8s_temp_dir", function_name).as_str());
@@ -284,13 +328,14 @@ WORKDIR {}
 
 # build and run project
 RUN RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo build{}
-CMD RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run{} 127.0.0.1:5000",
+CMD RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run{} 127.0.0.1:{}",
         tar_file_name,
         tar_file_name,
         tar_file_name,
         function_name,
         RELEASE_FLAG,
-        RELEASE_FLAG
+        RELEASE_FLAG,
+        INTERNAL_PORT
     );
     std::fs::write(&dockerfile_path, docker_file)?;
     std::fs::write(&tar_path, project_tar)?;
@@ -298,7 +343,7 @@ CMD RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run{} 127.0.0.1:5000",
     let result = (|| {
         // build image
         let build_cmd = format!(
-            "build -t {} {}",
+            "build -t {}:latest {}",
             function_name,
             build_dir_canonical.to_string_lossy()
         );
@@ -311,41 +356,46 @@ CMD RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run{} 127.0.0.1:5000",
             return Err(anyhow::anyhow!("docker image build failure"));
         }
 
-        let image_tag = format!(
-            "localhost:{}/{}",
-            registry_url.port().unwrap(),
-            function_name
-        );
+        // let image_tag = format!(
+        //     "localhost:{}/{}",
+        //     registry_url.port().unwrap(),
+        //     function_name
+        // );
 
         tracing::info!("made image tag");
 
         // tag image
-        let tag_args = format!("image tag {} {}", function_name, image_tag);
-        let tag_result = Command::new("docker")
-            .args(tag_args.as_str().split(' '))
-            .status()?;
-        if !tag_result.success() {
-            return Err(anyhow::anyhow!("docker image tag failure"));
-        }
-
-        tracing::info!("image tagged");
+        // let tag_args = format!("image tag {} {}", function_name, function_name);
+        // let tag_result = Command::new("docker")
+        //     .args(tag_args.as_str().split(' '))
+        //     .status()?;
+        // if !tag_result.success() {
+        //     return Err(anyhow::anyhow!("docker image tag failure"));
+        // }
+        // tracing::info!("image tagged");
 
         // push image to local repo
-        let push_status = Command::new("docker")
-            .arg("push")
-            .arg(image_tag.clone())
-            .status()?;
-        tracing::info!("docker push command did not explode");
-        if !push_status.success() {
-            return Err(anyhow::anyhow!("docker image push failure"));
-        }
+        // let image_tag = format!("dominicburkart/{}:latest", function_name.clone());
+        // let push_status = Command::new("docker")
+        //     .arg("push")
+        //     .arg(image_tag.clone())
+        //     .status()?;
+        // tracing::info!("docker push command did not explode");
+        // if !push_status.success() {
+        //     return Err(anyhow::anyhow!("docker image push failure"));
+        // }
 
-        Ok(image_tag)
+        // println!("haha >:D {}", image_tag);
+
+        Ok(function_name.into())
     })();
     tracing::info!("removing build dir");
     // always remove the build directory, even on build error
     std::fs::remove_dir_all(build_dir_canonical)?;
     tracing::info!("returning result");
+
+    Command::new("kind").args(format!("load docker-image {}", function_name).as_str().split(' ')).status()?;
+    // todo ^ temp fix while debugging kind
     result
 }
 
diff --git a/turbolift_macros/src/lib.rs b/turbolift_macros/src/lib.rs
index ca798342..950636be 100644
--- a/turbolift_macros/src/lib.rs
+++ b/turbolift_macros/src/lib.rs
@@ -32,7 +32,7 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
     let untyped_params = extract_function::to_untyped_params(typed_params.clone());
     let untyped_params_tokens = untyped_params.to_token_stream();
     let params_as_path = extract_function::to_path_params(untyped_params.clone());
-    let wrapper_route = "/".to_string() + &original_target_function_name + "/" + &params_as_path;
+    // let wrapper_route =  format!("/{}/{}", &app_name, &params_as_path);
     let param_types = extract_function::to_param_types(typed_params.clone());
     let params_vec = extract_function::params_json_vec(untyped_params.clone());
     let result_type = extract_function::get_result_type(&signature.output);
@@ -48,20 +48,24 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
     let sanitized_file = extract_function::get_sanitized_file(&function);
     // todo make code below hygienic in case sanitized_file also imports from actix_web
     let main_file = q! {
-        use turbolift::actix_web::{self, get, web, HttpResponse, Result};
+        use turbolift::actix_web::{self, get, web, HttpResponse, HttpRequest, Result, Responder};
         use turbolift::tokio_compat_02::FutureExt;
 
         #sanitized_file
         #dummy_function
         #target_function
 
-        #[get(#wrapper_route)]
-        #[turbolift::tracing::instrument]
-        async fn turbolift_wrapper(web::Path((#untyped_params_tokens)): web::Path<(#param_types)>) -> Result<HttpResponse> {
-            Ok(
-                HttpResponse::Ok()
-                    .json(#function_name(#untyped_params_tokens))
-            )
+        // #[get(#wrapper_route)]
+        // #[turbolift::tracing::instrument]
+        // async fn turbolift_wrapper(web::Path((#untyped_params_tokens)): web::Path<(#param_types)>) -> Result<HttpResponse> {
+        //     Ok(
+        //         HttpResponse::Ok()
+        //             .json(#function_name(#untyped_params_tokens))
+        //     )
+        // }
+
+        async fn return_path(req: HttpRequest) -> impl Responder {
+            HttpResponse::Ok().body(req.uri().to_string())
         }
 
         #[actix_web::main]
@@ -75,9 +79,13 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
             HttpServer::new(
                 ||
                     App::new()
-                        .service(
-                            turbolift_wrapper
+                        .default_service(
+                            web::resource("")
+                                .route(web::get().to(return_path))
                         )
+                        // .service(
+                        //     turbolift_wrapper
+                        // )
             )
             .bind(ip_and_port)?
             .run()

From 05ad5bc0b57bc5928764a0b1bcfc00994d7c7210 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Mon, 24 May 2021 20:16:22 +0200
Subject: [PATCH 093/145] debug bad gateway pt 1

---
 examples/kubernetes_example/Cargo.toml      |   2 +-
 examples/kubernetes_example/run_tests.sh    |   4 +-
 examples/local_queue_example/Cargo.toml     |   2 +-
 turbolift_internals/Cargo.toml              |  14 +-
 turbolift_internals/src/extract_function.rs |   2 +-
 turbolift_internals/src/kubernetes.rs       | 178 ++++++++++++--------
 turbolift_macros/src/lib.rs                 |   9 +-
 7 files changed, 127 insertions(+), 84 deletions(-)

diff --git a/examples/kubernetes_example/Cargo.toml b/examples/kubernetes_example/Cargo.toml
index 2f227eb9..52cf7bec 100644
--- a/examples/kubernetes_example/Cargo.toml
+++ b/examples/kubernetes_example/Cargo.toml
@@ -9,7 +9,7 @@ edition = "2018"
 
 [dependencies]
 rand = "0.7"
-tokio = {version="0.3", features=["full"]}
+tokio = {version="1", features=["full"]}
 lazy_static = "1"
 futures = "0.3"
 cute = "0.3"
diff --git a/examples/kubernetes_example/run_tests.sh b/examples/kubernetes_example/run_tests.sh
index 51692617..0f9cb95c 100644
--- a/examples/kubernetes_example/run_tests.sh
+++ b/examples/kubernetes_example/run_tests.sh
@@ -40,11 +40,11 @@ echo "cluster initialized."
 printf "\n🚪 adding ingress\n"
 kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/static/provider/kind/deploy.yaml
 sleep 90s # give k8s time to generate the pod ^
-printf "\n⏱️  waiting for ingress controller to be ready...\n"
+printf "\n⏱️ waiting for ingress controller to be ready...\n"
 kubectl wait --namespace ingress-nginx \
   --for=condition=ready pod \
   --selector=app.kubernetes.io/component=controller \
-  --timeout=30m 
+  --timeout=30m
 
 echo "🚪 ingress ready."
 
diff --git a/examples/local_queue_example/Cargo.toml b/examples/local_queue_example/Cargo.toml
index abcd6043..07b739a6 100644
--- a/examples/local_queue_example/Cargo.toml
+++ b/examples/local_queue_example/Cargo.toml
@@ -12,7 +12,7 @@ edition = "2018"
 rand = "0.7"
 futures = "0.3"
 lazy_static = "1"
-tokio = {version="0.3", features=["full"]}
+tokio = {version="1", features=["full"]}
 turbolift = { path="../../" }
 
 # for printing out tracing
diff --git a/turbolift_internals/Cargo.toml b/turbolift_internals/Cargo.toml
index 7bd1dcff..ea61b1b4 100644
--- a/turbolift_internals/Cargo.toml
+++ b/turbolift_internals/Cargo.toml
@@ -7,20 +7,20 @@ edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-syn = {version = "1", features=["full"] }
+syn = { version = "1", features=["full"] }
 quote = "1"
 serde = "1"
 serde_json = "1"
 brotli2 = "0.3.2"
 data-encoding = "2"
 futures = "0.3"
-proc-macro2 = { version = "1.0", features = ["span-locations"]}
+proc-macro2 = { version = "1", features = ["span-locations"]}
 tar = "0.4"
 toml = "0.5"
 cargo-toml2 = "1"
 tempfile = "3.1"
-reqwest = {repository="https://github.com/DominicBurkart/reqwest"}
-tokio = { version = "0.3", features = ["full"] }
+reqwest = "0.11"
+tokio = { version = "1", features = ["full"] }
 tokio-compat-02 = "0.1"
 cute = "0.3"
 rand = "0.7"
@@ -35,6 +35,6 @@ tracing = {version="0.1", features=["attributes"]}
 tracing-futures = "0.2.4"
 
 # kubernetes-specific requirements
-kube = "0.43.0"
-kube-runtime = "0.43.0"
-k8s-openapi = { version = "0.9.0", default-features = false, features = ["v1_17"] }
+kube = "0.51.0"
+kube-runtime = "0.51.0"
+k8s-openapi = { version = "0.11.0", default-features = false, features = ["v1_20"] }
diff --git a/turbolift_internals/src/extract_function.rs b/turbolift_internals/src/extract_function.rs
index 8bb33cd6..25de370e 100644
--- a/turbolift_internals/src/extract_function.rs
+++ b/turbolift_internals/src/extract_function.rs
@@ -5,8 +5,8 @@ use std::io::Cursor;
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
 
+use proc_macro2::TokenStream as TokenStream2;
 use quote::ToTokens;
-use syn::export::TokenStream2;
 use syn::spanned::Spanned;
 use tar::{Archive, Builder};
 
diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index f968ec5b..53fbf1a5 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -4,13 +4,13 @@ use std::str::FromStr;
 
 use async_trait::async_trait;
 use k8s_openapi::api::apps::v1::Deployment;
-// use k8s_openapi::api::core::v1::Service;
+use k8s_openapi::api::core::v1::Service;
 use kube::api::{Api, PostParams};
 use kube::Client;
 use regex::Regex;
+use tokio::time::{sleep, Duration};
 use tokio_compat_02::FutureExt;
 use url::Url;
-use tokio::time::{sleep, Duration};
 
 use crate::distributed_platform::{
     ArgsString, DistributionPlatform, DistributionResult, JsonResponse,
@@ -22,8 +22,9 @@ use std::io::Write;
 const TURBOLIFT_K8S_NAMESPACE: &str = "default";
 type ImageTag = String;
 
-pub const INTERNAL_PORT: i32 = 32766;
-pub const HOST_PORT: i32 = 80;
+pub const CONTAINER_PORT: i32 = 80;
+pub const SERVICE_PORT: i32 = 80;
+pub const EXTERNAL_PORT: i32 = 80;
 
 /// `K8s` is the interface for turning rust functions into autoscaling microservices
 /// using turbolift. It requires docker and kubernetes / kubectl to already be setup on the
@@ -79,15 +80,15 @@ impl DistributionPlatform for K8s {
         let deployment_client = Client::try_default().compat().await?;
         let deployments: Api<Deployment> =
             Api::namespaced(deployment_client, TURBOLIFT_K8S_NAMESPACE);
-        // let service_client = Client::try_default().compat().await?;
-        // let services: Api<Service> = Api::namespaced(service_client, TURBOLIFT_K8S_NAMESPACE);
+        let service_client = Client::try_default().compat().await?;
+        let services: Api<Service> = Api::namespaced(service_client, TURBOLIFT_K8S_NAMESPACE);
 
         // generate image & push
-        let tag_in_reg = make_image(function_name, project_tar)?;
+        let app_name = function_to_app_name(function_name);
+        let tag_in_reg = make_image(&app_name, project_tar)?;
 
         println!("image made. making deployment and service names.");
         println!("made service_name");
-        let app_name = function_to_app_name(function_name);
         println!("made app_name and container_name");
 
         // make deployment
@@ -107,7 +108,7 @@ impl DistributionPlatform for K8s {
                 "template": {
                     "metadata": {
                         "labels": {
-                            "run": app_name
+                            "run": app_name,
                         }
                     },
                     "spec": {
@@ -118,7 +119,7 @@ impl DistributionPlatform for K8s {
                                 "imagePullPolicy": "IfNotPresent", // prefer local image
                                 "ports": [
                                     {
-                                        "containerPort": INTERNAL_PORT
+                                        "containerPort": CONTAINER_PORT
                                     }
                                 ]
                             },
@@ -137,52 +138,81 @@ impl DistributionPlatform for K8s {
         println!("created deployment");
 
         // make service pointing to deployment
-       if !Command::new("kubectl")
-           .args(format!("expose deployment/{}", app_name).as_str().split(' '))
-           .status()?
-           .success() {
-           tracing::error!("could not make service");
-           panic!("could not make service to expose deployment");
-       }
-
-        tracing::info!("created service");
+        if !Command::new("kubectl")
+            .args(
+                format!("expose deployment/{}", app_name)
+                    .as_str()
+                    .split(' '),
+            )
+            .status()?
+            .success()
+        {
+            tracing::error!("could not make service");
+            panic!("could not make service to expose deployment");
+        }
+        let service_json = serde_json::json!({
+            "apiVersion": "v1",
+            "kind": "Service",
+            "metadata": {
+                "name": app_name
+            },
+            "spec": {
+                "selector": {
+                    "run": app_name
+                },
+                "ports": [{
+                    "port": SERVICE_PORT,
+                    "targetPort": CONTAINER_PORT,
+                }]
+            }
+        });
+        let service = serde_json::from_value(service_json)?;
+        println!("deployment generated");
+        services
+            .create(&PostParams::default(), &service)
+            .compat()
+            .await?;
+        println!("created service");
 
         // make ingress pointing to service
         let ingress = serde_json::json!({
             "apiVersion": "networking.k8s.io/v1",
             "kind": "Ingress",
             "metadata": {
-                "name": app_name,
+                "name": app_name
+                // "annotations": {
+                //     "nginx.ingress.kubernetes.io/rewrite-target": "/"
+                // }
             },
             "spec": {
-                // "defaultBackend": {
-                //     "service": {
-                //         "name": app_name,
-                //         "port": {
-                //             "number": PORT
-                //         }
-                //     }
-                // }
-                "rules": [
-                    {
-                        "http": {
-                            "paths": [
-                                {
-                                    "path": format!("/{}", app_name),
-                                    "pathType": "Prefix",
-                                    "backend": {
-                                        "service": {
-                                            "name": app_name,
-                                            "port": {
-                                                "number": INTERNAL_PORT
-                                            }
-                                        }
-                                    }
-                                }
-                            ]
+                "defaultBackend": {
+                    "service": {
+                        "name": app_name,
+                        "port": {
+                            "number": SERVICE_PORT
                         }
                     }
-                ]
+                }
+                // "rules": [
+                //     {
+                //         "http": {
+                //             "paths": [
+                //                 {
+                //                     "path": format!("/{}", app_name),
+                //                     "pathType": "Prefix",
+                //                     "backend": {
+                //                         "service" : {
+                //                             "name": app_name,
+                //                             "port": {
+                //                                 "number": SERVICE_PORT
+                //                             }
+                //                         }
+                //                     }
+                //                 }
+                //             ]
+                //         }
+                //     }
+                // ]
             }
         });
 
@@ -196,7 +226,10 @@ impl DistributionPlatform for K8s {
             .expect("not able to write to ingress apply stdin")
             .write_all(ingress.to_string().as_bytes())?;
         if !apply_ingress_child.wait()?.success() {
-            panic!("failed to apply ingress: {}\nis ingress enabled on this cluster?", ingress.to_string())
+            panic!(
+                "failed to apply ingress: {}\nis ingress enabled on this cluster?",
+                ingress.to_string()
+            )
         }
 
         let node_ip = {
@@ -208,7 +241,7 @@ impl DistributionPlatform for K8s {
             // String::from_utf8(stdout).expect("could not parse local node ip")
             "localhost".to_string()
         };
-        tracing::info!(node_ip = node_ip.as_str(), "found node ip");
+        println!("found node ip: {}", node_ip.as_str());
 
         // let node_port: i32 = 5000;
         // let node_port = service
@@ -221,8 +254,8 @@ impl DistributionPlatform for K8s {
         //     .next()
         //     .expect("no node port assigned to service");
         // let service_ip = format!("http://{}", node_ip, node_port);
-        let service_ip = format!("http://localhost:{}/{}/", HOST_PORT, app_name);
-        tracing::info!(ip = service_ip.as_str(), "generated service_ip");
+        let service_ip = format!("http://localhost:{}/{}/", EXTERNAL_PORT, app_name);
+        println!("generated service_ip: {}", service_ip.as_str());
 
         // todo make sure that the pod and service were correctly started before returning
 
@@ -252,6 +285,8 @@ impl DistributionPlatform for K8s {
         self.fn_names_to_services
             .insert(function_name.to_string(), Url::from_str(&service_ip)?);
         // todo handle deleting the relevant service and deployment for each distributed function.
+
+        println!("returning from declare");
         Ok(())
     }
 
@@ -263,9 +298,10 @@ impl DistributionPlatform for K8s {
     ) -> DistributionResult<JsonResponse> {
         // request from server
         let service_base_url = self.fn_names_to_services.get(function_name).unwrap();
-        let args = "./".to_string() + function_name + "/" + &params;
+        let args = format!("./{}", params);
         let query_url = service_base_url.join(&args)?;
         tracing::info!(url = query_url.as_str(), "sending dispatch request");
+        println!("sending dispatch request");
         Ok(self
             .request_client
             .get(query_url)
@@ -279,6 +315,7 @@ impl DistributionPlatform for K8s {
 
     #[tracing::instrument]
     fn has_declared(&self, fn_name: &str) -> bool {
+        println!("in has_declared");
         self.fn_names_to_services.contains_key(fn_name)
     }
 }
@@ -288,17 +325,13 @@ lazy_static! {
 }
 
 #[tracing::instrument(skip(project_tar))]
-fn make_image(
-    function_name: &str,
-    project_tar: &[u8],
-) -> anyhow::Result<ImageTag> {
+fn make_image(app_name: &str, project_tar: &[u8]) -> anyhow::Result<ImageTag> {
     // todo: we should add some random stuff to the function_name to avoid collisions and figure
     // out when to overwrite vs not.
 
-
     tracing::info!("making image");
     // set up directory and dockerfile
-    let build_dir = CACHE_PATH.join(format!("{}_k8s_temp_dir", function_name).as_str());
+    let build_dir = CACHE_PATH.join(format!("{}_k8s_temp_dir", app_name).as_str());
     std::fs::create_dir_all(&build_dir)?;
     let build_dir_canonical = build_dir.canonicalize()?;
     let dockerfile_path = build_dir_canonical.join("Dockerfile");
@@ -314,7 +347,7 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 RUN apt-get update && apt-get install -y curl gcc libssl-dev pkg-config && rm -rf /var/lib/apt/lists/*
 
 # install rustup
-RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly-2020-09-28
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly
 ENV PATH=/root/.cargo/bin:$PATH
 
 # copy tar file
@@ -332,19 +365,20 @@ CMD RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run{} 127.0.0.1:{}",
         tar_file_name,
         tar_file_name,
         tar_file_name,
-        function_name,
+        app_name,
         RELEASE_FLAG,
         RELEASE_FLAG,
-        INTERNAL_PORT
+        CONTAINER_PORT
     );
     std::fs::write(&dockerfile_path, docker_file)?;
     std::fs::write(&tar_path, project_tar)?;
+    let unique_tag = format!("{}:turbolift", app_name); // todo find better unique tag
 
     let result = (|| {
         // build image
         let build_cmd = format!(
-            "build -t {}:latest {}",
-            function_name,
+            "build -t {} {}",
+            unique_tag,
             build_dir_canonical.to_string_lossy()
         );
         let build_status = Command::new("docker")
@@ -362,7 +396,7 @@ CMD RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run{} 127.0.0.1:{}",
         //     function_name
         // );
 
-        tracing::info!("made image tag");
+        println!("made image tag");
 
         // tag image
         // let tag_args = format!("image tag {} {}", function_name, function_name);
@@ -387,14 +421,20 @@ CMD RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run{} 127.0.0.1:{}",
 
         // println!("haha >:D {}", image_tag);
 
-        Ok(function_name.into())
+        Ok(unique_tag.clone())
     })();
-    tracing::info!("removing build dir");
+    println!("removing build dir");
     // always remove the build directory, even on build error
     std::fs::remove_dir_all(build_dir_canonical)?;
-    tracing::info!("returning result");
-
-    Command::new("kind").args(format!("load docker-image {}", function_name).as_str().split(' ')).status()?;
+    println!("returning result");
+
+    Command::new("kind")
+        .args(
+            format!("load docker-image {}", unique_tag)
+                .as_str()
+                .split(' '),
+        )
+        .status()?;
     // todo ^ temp fix while debugging kind
     result
 }
@@ -402,6 +442,8 @@ CMD RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run{} 127.0.0.1:{}",
 impl Drop for K8s {
     #[tracing::instrument]
     fn drop(&mut self) {
+        // todo
+
         // delete the associated services and deployments from the functions we distributed
         // let rt = tokio::runtime::Runtime::new().unwrap();
         // rt.block_on(async {
diff --git a/turbolift_macros/src/lib.rs b/turbolift_macros/src/lib.rs
index 950636be..8f032421 100644
--- a/turbolift_macros/src/lib.rs
+++ b/turbolift_macros/src/lib.rs
@@ -197,17 +197,17 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
             turbolift::tracing::info!("platform acquired");
 
             if !platform.has_declared(#original_target_function_name) {
-                turbolift::tracing::info!("launching declare");
+                println!("launching declare");
                 platform
                     .declare(#original_target_function_name, #project_source_binary)
                     .compat()
                     .await?;
-                turbolift::tracing::info!("declare completed");
+                println!("declare completed");
             }
 
             let params = #params_vec.join("/");
 
-            turbolift::tracing::info!("launching dispatch");
+            println!("launching dispatch");
             let resp_string = platform
                 .dispatch(
                     #original_target_function_name,
@@ -215,7 +215,8 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
                 )
                 .compat()
                 .await?;
-            turbolift::tracing::info!("dispatch completed");
+            println!("dispatch completed");
+            println!("resp_string: {}", &resp_string);
             Ok(turbolift::serde_json::from_str(&resp_string)?)
         }
     };

From 22b52b611d815d4339e87e24ac1af2d76df651d6 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Thu, 27 May 2021 19:17:40 +0200
Subject: [PATCH 094/145] remove debug code causing service to return source
 path

---
 turbolift_internals/src/local_queue.rs |  2 +-
 turbolift_macros/src/lib.rs            | 27 ++++++++++++--------------
 2 files changed, 13 insertions(+), 16 deletions(-)

diff --git a/turbolift_internals/src/local_queue.rs b/turbolift_internals/src/local_queue.rs
index 6cc204aa..274960ee 100644
--- a/turbolift_internals/src/local_queue.rs
+++ b/turbolift_internals/src/local_queue.rs
@@ -103,7 +103,7 @@ impl DistributionPlatform for LocalQueue {
             .text()
             .compat()
             .await?);
-        tracing::info!("received response");
+        println!("received response");
         resp
     }
 
diff --git a/turbolift_macros/src/lib.rs b/turbolift_macros/src/lib.rs
index 8f032421..e4fda689 100644
--- a/turbolift_macros/src/lib.rs
+++ b/turbolift_macros/src/lib.rs
@@ -32,7 +32,8 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
     let untyped_params = extract_function::to_untyped_params(typed_params.clone());
     let untyped_params_tokens = untyped_params.to_token_stream();
     let params_as_path = extract_function::to_path_params(untyped_params.clone());
-    // let wrapper_route =  format!("/{}/{}", &app_name, &params_as_path);
+    let wrapper_route = format!("/{}/{}", &original_target_function_name, &params_as_path);
+    // ^ todo: make this unique even if multiple functions with the same name are distributed at the same time
     let param_types = extract_function::to_param_types(typed_params.clone());
     let params_vec = extract_function::params_json_vec(untyped_params.clone());
     let result_type = extract_function::get_result_type(&signature.output);
@@ -55,14 +56,14 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
         #dummy_function
         #target_function
 
-        // #[get(#wrapper_route)]
-        // #[turbolift::tracing::instrument]
-        // async fn turbolift_wrapper(web::Path((#untyped_params_tokens)): web::Path<(#param_types)>) -> Result<HttpResponse> {
-        //     Ok(
-        //         HttpResponse::Ok()
-        //             .json(#function_name(#untyped_params_tokens))
-        //     )
-        // }
+        #[get(#wrapper_route)]
+        #[turbolift::tracing::instrument]
+        async fn turbolift_wrapper(web::Path((#untyped_params_tokens)): web::Path<(#param_types)>) -> Result<HttpResponse> {
+            Ok(
+                HttpResponse::Ok()
+                    .json(#function_name(#untyped_params_tokens))
+            )
+        }
 
         async fn return_path(req: HttpRequest) -> impl Responder {
             HttpResponse::Ok().body(req.uri().to_string())
@@ -79,13 +80,9 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
             HttpServer::new(
                 ||
                     App::new()
-                        .default_service(
-                            web::resource("")
-                                .route(web::get().to(return_path))
+                        .service(
+                            turbolift_wrapper
                         )
-                        // .service(
-                        //     turbolift_wrapper
-                        // )
             )
             .bind(ip_and_port)?
             .run()

From 65f10cdd2a502692aa825cbc16d549094ac7ff44 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Thu, 27 May 2021 20:25:52 +0200
Subject: [PATCH 095/145] remove duplicate service creation command

---
 turbolift_internals/src/kubernetes.rs | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 53fbf1a5..55b6e741 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -138,18 +138,6 @@ impl DistributionPlatform for K8s {
         println!("created deployment");
 
         // make service pointing to deployment
-        if !Command::new("kubectl")
-            .args(
-                format!("expose deployment/{}", app_name)
-                    .as_str()
-                    .split(' '),
-            )
-            .status()?
-            .success()
-        {
-            tracing::error!("could not make service");
-            panic!("could not make service to expose deployment");
-        }
         let service_json = serde_json::json!({
             "apiVersion": "v1",
             "kind": "Service",

From de757c7d2f2d7b4df365e2aad797092cca59f2cd Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Fri, 28 May 2021 21:51:56 +0200
Subject: [PATCH 096/145] check if changing ingress api version fixes ingress
 issue in ci

---
 turbolift_internals/src/kubernetes.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 55b6e741..d4c2aedc 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -164,7 +164,7 @@ impl DistributionPlatform for K8s {
 
         // make ingress pointing to service
         let ingress = serde_json::json!({
-            "apiVersion": "networking.k8s.io/v1",
+            "apiVersion": "v1",
             "kind": "Ingress",
             "metadata": {
                 "name": app_name

From c8a84d371b41b2a6492ffec825cc1960f23bdd22 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sat, 5 Jun 2021 12:28:19 +0200
Subject: [PATCH 097/145] debug bad gateway using pod instead of deployment

---
 turbolift_internals/src/kubernetes.rs | 191 +++++++++++++++-----------
 1 file changed, 109 insertions(+), 82 deletions(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index d4c2aedc..5053e769 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -3,8 +3,8 @@ use std::process::{Command, Stdio};
 use std::str::FromStr;
 
 use async_trait::async_trait;
-use k8s_openapi::api::apps::v1::Deployment;
-use k8s_openapi::api::core::v1::Service;
+// use k8s_openapi::api::apps::v1::Deployment;
+use k8s_openapi::api::core::v1::{Pod, Service};
 use kube::api::{Api, PostParams};
 use kube::Client;
 use regex::Regex;
@@ -22,8 +22,8 @@ use std::io::Write;
 const TURBOLIFT_K8S_NAMESPACE: &str = "default";
 type ImageTag = String;
 
-pub const CONTAINER_PORT: i32 = 80;
-pub const SERVICE_PORT: i32 = 80;
+pub const CONTAINER_PORT: i32 = 5678;
+pub const SERVICE_PORT: i32 = 5678;
 pub const EXTERNAL_PORT: i32 = 80;
 
 /// `K8s` is the interface for turning rust functions into autoscaling microservices
@@ -77,80 +77,103 @@ impl DistributionPlatform for K8s {
     #[tracing::instrument(skip(project_tar))]
     async fn declare(&mut self, function_name: &str, project_tar: &[u8]) -> DistributionResult<()> {
         // connect to cluster. tries in-cluster configuration first, then falls back to kubeconfig file.
-        let deployment_client = Client::try_default().compat().await?;
-        let deployments: Api<Deployment> =
-            Api::namespaced(deployment_client, TURBOLIFT_K8S_NAMESPACE);
+        let pod_client = Client::try_default().compat().await?;
+        let pods: Api<Pod> = Api::namespaced(pod_client, TURBOLIFT_K8S_NAMESPACE);
+        // let deployment_client = Client::try_default().compat().await?;
+        // let deployments: Api<Deployment> =
+        //     Api::namespaced(deployment_client, TURBOLIFT_K8S_NAMESPACE);
         let service_client = Client::try_default().compat().await?;
         let services: Api<Service> = Api::namespaced(service_client, TURBOLIFT_K8S_NAMESPACE);
 
         // generate image & push
         let app_name = function_to_app_name(function_name);
+        let service_name = format!("{}-service", app_name);
         let tag_in_reg = make_image(&app_name, project_tar)?;
 
         println!("image made. making deployment and service names.");
         println!("made service_name");
         println!("made app_name and container_name");
-
-        // make deployment
-        let deployment_json = serde_json::json!({
-            "apiVersion": "apps/v1",
-            "kind": "Deployment",
-            "metadata": {
-                "name": app_name,
-            },
-            "spec": {
-                "selector": {
-                    "matchLabels": {
-                        "run": app_name
-                    }
-                },
-                "replicas": 1,
-                "template": {
-                    "metadata": {
-                        "labels": {
-                            "run": app_name,
-                        }
-                    },
-                    "spec": {
-                        "containers": [
-                            {
-                                "name": app_name,
-                                "image": tag_in_reg,
-                                "imagePullPolicy": "IfNotPresent", // prefer local image
-                                "ports": [
-                                    {
-                                        "containerPort": CONTAINER_PORT
-                                    }
-                                ]
-                            },
-                        ]
-                    }
+        let pod_json = serde_json::json!({
+           "kind": "Pod",
+           "apiVersion":"v1",
+           "metadata": {
+             "name": format!("{}-app", app_name),
+             "labels": {
+               "app": app_name
+             }
+           },
+           "spec": {
+             "containers": [
+                {
+                    "name": format!("{}-app", app_name),
+                    "image": tag_in_reg
                 }
-            }
+             ]
+           }
         });
-        println!("deployment_json generated");
-        let deployment = serde_json::from_value(deployment_json)?;
-        println!("deployment generated");
-        deployments
-            .create(&PostParams::default(), &deployment)
-            .compat()
-            .await?;
-        println!("created deployment");
+        let pod = serde_json::from_value(pod_json)?;
+        pods.create(&PostParams::default(), &pod).compat().await?;
+
+        // // make deployment
+        // let deployment_json = serde_json::json!({
+        //     "apiVersion": "apps/v1",
+        //     "kind": "Deployment",
+        //     "metadata": {
+        //         "name": app_name,
+        //     },
+        //     "spec": {
+        //         "selector": {
+        //             "matchLabels": {
+        //                 "run": app_name
+        //             }
+        //         },
+        //         "replicas": 1,
+        //         "template": {
+        //             "metadata": {
+        //                 "labels": {
+        //                     "run": app_name,
+        //                 }
+        //             },
+        //             "spec": {
+        //                 "containers": [
+        //                     {
+        //                         "name": app_name,
+        //                         "image": tag_in_reg,
+        //                         "imagePullPolicy": "IfNotPresent", // prefer local image
+        //                         "ports": [
+        //                             {
+        //                                 "containerPort": CONTAINER_PORT
+        //                             }
+        //                         ]
+        //                     },
+        //                 ]
+        //             }
+        //         }
+        //     }
+        // });
+        // println!("deployment_json generated");
+        // let deployment = serde_json::from_value(deployment_json)?;
+        // println!("deployment generated");
+        // deployments
+        //     .create(&PostParams::default(), &deployment)
+        //     .compat()
+        //     .await?;
+        // println!("created deployment");
 
         // make service pointing to deployment
         let service_json = serde_json::json!({
             "apiVersion": "v1",
             "kind": "Service",
             "metadata": {
-                "name": app_name
+                "name": service_name,
             },
             "spec": {
                 "selector": {
-                    "run": app_name
+                    "app": app_name
                 },
                 "ports": [{
                     "port": SERVICE_PORT,
-                    "targetPort": CONTAINER_PORT,
+                    // "targetPort": CONTAINER_PORT,
                 }]
             }
         });
@@ -164,43 +187,47 @@ impl DistributionPlatform for K8s {
 
         // make ingress pointing to service
         let ingress = serde_json::json!({
-            "apiVersion": "v1",
+            "apiVersion": "networking.k8s.io/v1beta1",
             "kind": "Ingress",
             "metadata": {
-                "name": app_name
+                "name": format!("{}-ingress", app_name),
                 // "annotations": {
                 //     "nginx.ingress.kubernetes.io/rewrite-target": "/"
                 // }
             },
             "spec": {
-                "defaultBackend": {
-                    "service": {
-                        "name": app_name,
-                        "port": {
-                            "number": SERVICE_PORT
-                        }
-                    }
-                }
-                // "rules": [
-                //     {
-                //         "http": {
-                //             "paths": [
-                //                 {
-                //                     "path": format!("/{}", app_name),
-                //                     "pathType": "Prefix",
-                //                     "backend": {
-                //                         "service" : {
-                //                             "name": app_name,
-                //                             "port": {
-                //                                 "number": SERVICE_PORT
-                //                             }
-                //                         }
-                //                     }
-                //                 }
-                //             ]
+                // "defaultBackend": {
+                //     "service": {
+                //         "name": app_name,
+                //         "port": {
+                //             "number": SERVICE_PORT
                 //         }
                 //     }
-                // ]
+                // }
+                "rules": [
+                    {
+                        "http": {
+                            "paths": [
+                                {
+                                    "path": format!("/{}", app_name),
+                                    // "pathType": "Prefix",
+                                    // "backend": {
+                                    //     "service" : {
+                                    //         "name": app_name,
+                                    //         "port": {
+                                    //             "number": SERVICE_PORT
+                                    //         }
+                                    //     }
+                                    // }
+                                    "backend": {
+                                        "serviceName": app_name,
+                                        "servicePort": SERVICE_PORT
+                                    }
+                                }
+                            ]
+                        }
+                    }
+                ]
             }
         });
 

From 4dada05be2130fb3f253927063da21dec0947fd2 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sat, 5 Jun 2021 18:51:38 +0200
Subject: [PATCH 098/145] fix binding to the wrong port in the app container

---
 turbolift_internals/src/kubernetes.rs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 5053e769..c40390de 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -220,7 +220,7 @@ impl DistributionPlatform for K8s {
                                     //     }
                                     // }
                                     "backend": {
-                                        "serviceName": app_name,
+                                        "serviceName": service_name,
                                         "servicePort": SERVICE_PORT
                                     }
                                 }
@@ -375,8 +375,9 @@ RUN cat {} | tar xvf -
 WORKDIR {}
 
 # build and run project
-RUN RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo build{}
-CMD RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run{} 127.0.0.1:{}",
+ENV RUSTFLAGS='--cfg procmacro2_semver_exempt'
+RUN cargo build{}
+CMD cargo run{} 0.0.0.0:{}",
         tar_file_name,
         tar_file_name,
         tar_file_name,

From f17e11b938c44e6039f6457fdb3276305f7be3d6 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sat, 5 Jun 2021 19:30:56 +0200
Subject: [PATCH 099/145] use deployment instead of pod

---
 turbolift_internals/src/kubernetes.rs | 133 ++++++++++----------------
 1 file changed, 49 insertions(+), 84 deletions(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index c40390de..d2039396 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -3,8 +3,8 @@ use std::process::{Command, Stdio};
 use std::str::FromStr;
 
 use async_trait::async_trait;
-// use k8s_openapi::api::apps::v1::Deployment;
-use k8s_openapi::api::core::v1::{Pod, Service};
+use k8s_openapi::api::apps::v1::Deployment;
+use k8s_openapi::api::core::v1::Service;
 use kube::api::{Api, PostParams};
 use kube::Client;
 use regex::Regex;
@@ -77,88 +77,64 @@ impl DistributionPlatform for K8s {
     #[tracing::instrument(skip(project_tar))]
     async fn declare(&mut self, function_name: &str, project_tar: &[u8]) -> DistributionResult<()> {
         // connect to cluster. tries in-cluster configuration first, then falls back to kubeconfig file.
-        let pod_client = Client::try_default().compat().await?;
-        let pods: Api<Pod> = Api::namespaced(pod_client, TURBOLIFT_K8S_NAMESPACE);
-        // let deployment_client = Client::try_default().compat().await?;
-        // let deployments: Api<Deployment> =
-        //     Api::namespaced(deployment_client, TURBOLIFT_K8S_NAMESPACE);
+        let deployment_client = Client::try_default().compat().await?;
+        let deployments: Api<Deployment> =
+            Api::namespaced(deployment_client, TURBOLIFT_K8S_NAMESPACE);
         let service_client = Client::try_default().compat().await?;
         let services: Api<Service> = Api::namespaced(service_client, TURBOLIFT_K8S_NAMESPACE);
 
         // generate image & push
         let app_name = function_to_app_name(function_name);
+        let container_name = format!("{}-app", app_name);
+        let deployment_name = format!("{}-deployment", app_name);
         let service_name = format!("{}-service", app_name);
+        let ingress_name = format!("{}-ingress", app_name);
         let tag_in_reg = make_image(&app_name, project_tar)?;
 
         println!("image made. making deployment and service names.");
         println!("made service_name");
         println!("made app_name and container_name");
-        let pod_json = serde_json::json!({
-           "kind": "Pod",
-           "apiVersion":"v1",
-           "metadata": {
-             "name": format!("{}-app", app_name),
-             "labels": {
-               "app": app_name
-             }
-           },
-           "spec": {
-             "containers": [
-                {
-                    "name": format!("{}-app", app_name),
-                    "image": tag_in_reg
+
+        // make deployment
+        let deployment_json = serde_json::json!({
+            "apiVersion": "apps/v1",
+            "kind": "Deployment",
+            "metadata": {
+                "name": deployment_name,
+            },
+            "spec": {
+                "selector": {
+                    "matchLabels": {
+                        "app": app_name
+                    }
+                },
+                "replicas": 1,
+                "template": {
+                    "metadata": {
+                     "name": format!("{}-app", app_name),
+                     "labels": {
+                       "app": app_name
+                     }
+                    },
+                    "spec": {
+                        "containers": [
+                            {
+                                "name": container_name,
+                                "image": tag_in_reg
+                            }
+                         ]
+                    }
                 }
-             ]
-           }
+            }
         });
-        let pod = serde_json::from_value(pod_json)?;
-        pods.create(&PostParams::default(), &pod).compat().await?;
-
-        // // make deployment
-        // let deployment_json = serde_json::json!({
-        //     "apiVersion": "apps/v1",
-        //     "kind": "Deployment",
-        //     "metadata": {
-        //         "name": app_name,
-        //     },
-        //     "spec": {
-        //         "selector": {
-        //             "matchLabels": {
-        //                 "run": app_name
-        //             }
-        //         },
-        //         "replicas": 1,
-        //         "template": {
-        //             "metadata": {
-        //                 "labels": {
-        //                     "run": app_name,
-        //                 }
-        //             },
-        //             "spec": {
-        //                 "containers": [
-        //                     {
-        //                         "name": app_name,
-        //                         "image": tag_in_reg,
-        //                         "imagePullPolicy": "IfNotPresent", // prefer local image
-        //                         "ports": [
-        //                             {
-        //                                 "containerPort": CONTAINER_PORT
-        //                             }
-        //                         ]
-        //                     },
-        //                 ]
-        //             }
-        //         }
-        //     }
-        // });
-        // println!("deployment_json generated");
-        // let deployment = serde_json::from_value(deployment_json)?;
-        // println!("deployment generated");
-        // deployments
-        //     .create(&PostParams::default(), &deployment)
-        //     .compat()
-        //     .await?;
-        // println!("created deployment");
+        println!("deployment_json generated");
+        let deployment = serde_json::from_value(deployment_json)?;
+        println!("deployment generated");
+        deployments
+            .create(&PostParams::default(), &deployment)
+            .compat()
+            .await?;
+        println!("created deployment");
 
         // make service pointing to deployment
         let service_json = serde_json::json!({
@@ -173,7 +149,7 @@ impl DistributionPlatform for K8s {
                 },
                 "ports": [{
                     "port": SERVICE_PORT,
-                    // "targetPort": CONTAINER_PORT,
+                    "targetPort": CONTAINER_PORT,
                 }]
             }
         });
@@ -190,20 +166,9 @@ impl DistributionPlatform for K8s {
             "apiVersion": "networking.k8s.io/v1beta1",
             "kind": "Ingress",
             "metadata": {
-                "name": format!("{}-ingress", app_name),
-                // "annotations": {
-                //     "nginx.ingress.kubernetes.io/rewrite-target": "/"
-                // }
+                "name": ingress_name
             },
             "spec": {
-                // "defaultBackend": {
-                //     "service": {
-                //         "name": app_name,
-                //         "port": {
-                //             "number": SERVICE_PORT
-                //         }
-                //     }
-                // }
                 "rules": [
                     {
                         "http": {

From 4c7f4a86134d50369620ace6f873200c835e18b9 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sat, 5 Jun 2021 19:49:25 +0200
Subject: [PATCH 100/145] use stable API

---
 turbolift_internals/src/kubernetes.rs | 19 +++++++------------
 1 file changed, 7 insertions(+), 12 deletions(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index d2039396..2b8ff867 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -163,7 +163,7 @@ impl DistributionPlatform for K8s {
 
         // make ingress pointing to service
         let ingress = serde_json::json!({
-            "apiVersion": "networking.k8s.io/v1beta1",
+            "apiVersion": "networking.k8s.io/v1",
             "kind": "Ingress",
             "metadata": {
                 "name": ingress_name
@@ -175,18 +175,13 @@ impl DistributionPlatform for K8s {
                             "paths": [
                                 {
                                     "path": format!("/{}", app_name),
-                                    // "pathType": "Prefix",
-                                    // "backend": {
-                                    //     "service" : {
-                                    //         "name": app_name,
-                                    //         "port": {
-                                    //             "number": SERVICE_PORT
-                                    //         }
-                                    //     }
-                                    // }
                                     "backend": {
-                                        "serviceName": service_name,
-                                        "servicePort": SERVICE_PORT
+                                        "service" : {
+                                            "name": service_name,
+                                            "port": {
+                                                "number": SERVICE_PORT
+                                            }
+                                        }
                                     }
                                 }
                             ]

From c39fcaaa0dee3c1693000e452ce5e876c4b0c1a2 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sat, 5 Jun 2021 20:20:40 +0200
Subject: [PATCH 101/145] don't send .turbolift files in project jars

---
 turbolift_internals/src/extract_function.rs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/turbolift_internals/src/extract_function.rs b/turbolift_internals/src/extract_function.rs
index 25de370e..2d927e59 100644
--- a/turbolift_internals/src/extract_function.rs
+++ b/turbolift_internals/src/extract_function.rs
@@ -189,8 +189,10 @@ pub fn make_compressed_proj_src(dir: &Path) -> Vec<u8> {
             if entry.path().is_dir() {
                 // ^ bug: the metadata on symlinks sometimes say that they are not directories,
                 // so we have to metadata.is_dir() || (metadata.file_type().is_symlink() && entry.path().is_dir() )
-                if Some("target") == entry.file_name().to_str() {
-                    // don't include any target files
+                if Some("target") == entry.file_name().to_str()
+                    || Some(".turbolift") == entry.file_name().to_str()
+                {
+                    // don't include any target or .turbolift directories
                 } else {
                     archive
                         .append_dir(&entry_path_with_parent, entry.path())

From c7aaf751be1cacc627c2109b8a7d3a26f56fb363 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sat, 5 Jun 2021 20:23:34 +0200
Subject: [PATCH 102/145] Revert "use stable API"

This reverts commit 4c7f4a86134d50369620ace6f873200c835e18b9.
---
 turbolift_internals/src/kubernetes.rs | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 2b8ff867..d2039396 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -163,7 +163,7 @@ impl DistributionPlatform for K8s {
 
         // make ingress pointing to service
         let ingress = serde_json::json!({
-            "apiVersion": "networking.k8s.io/v1",
+            "apiVersion": "networking.k8s.io/v1beta1",
             "kind": "Ingress",
             "metadata": {
                 "name": ingress_name
@@ -175,13 +175,18 @@ impl DistributionPlatform for K8s {
                             "paths": [
                                 {
                                     "path": format!("/{}", app_name),
+                                    // "pathType": "Prefix",
+                                    // "backend": {
+                                    //     "service" : {
+                                    //         "name": app_name,
+                                    //         "port": {
+                                    //             "number": SERVICE_PORT
+                                    //         }
+                                    //     }
+                                    // }
                                     "backend": {
-                                        "service" : {
-                                            "name": service_name,
-                                            "port": {
-                                                "number": SERVICE_PORT
-                                            }
-                                        }
+                                        "serviceName": service_name,
+                                        "servicePort": SERVICE_PORT
                                     }
                                 }
                             ]

From c2d2470e7cd8ec729b00dcc6993a8ffa1f586369 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sat, 5 Jun 2021 20:59:05 +0200
Subject: [PATCH 103/145] explicitly set pathType

---
 turbolift_internals/src/kubernetes.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index d2039396..3d7572f0 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -175,7 +175,7 @@ impl DistributionPlatform for K8s {
                             "paths": [
                                 {
                                     "path": format!("/{}", app_name),
-                                    // "pathType": "Prefix",
+                                    "pathType": "Prefix",
                                     // "backend": {
                                     //     "service" : {
                                     //         "name": app_name,

From 76f13051741b1353c80ccf35da211b53ead9e023 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sat, 5 Jun 2021 23:47:16 +0200
Subject: [PATCH 104/145] at runtime, turbolift should generate a unique id per
 run

---
 turbolift_internals/Cargo.toml                 |  1 +
 .../src/distributed_platform.rs                |  8 +++++++-
 turbolift_internals/src/kubernetes.rs          | 18 ++++++++++++++----
 turbolift_internals/src/lib.rs                 |  1 +
 turbolift_internals/src/local_queue.rs         | 15 ++++++++++++---
 turbolift_macros/src/lib.rs                    |  3 ++-
 6 files changed, 37 insertions(+), 9 deletions(-)

diff --git a/turbolift_internals/Cargo.toml b/turbolift_internals/Cargo.toml
index ea61b1b4..2ad24065 100644
--- a/turbolift_internals/Cargo.toml
+++ b/turbolift_internals/Cargo.toml
@@ -33,6 +33,7 @@ get_if_addrs = "0.5.3"
 regex = "1"
 tracing = {version="0.1", features=["attributes"]}
 tracing-futures = "0.2.4"
+uuid = { version="0.8", features=["v4"] }
 
 # kubernetes-specific requirements
 kube = "0.51.0"
diff --git a/turbolift_internals/src/distributed_platform.rs b/turbolift_internals/src/distributed_platform.rs
index ba0bfce7..59944b88 100644
--- a/turbolift_internals/src/distributed_platform.rs
+++ b/turbolift_internals/src/distributed_platform.rs
@@ -1,6 +1,7 @@
 extern crate proc_macro;
 use async_trait::async_trait;
 use std::error;
+use uuid::Uuid;
 
 pub type DistributionError = Box<dyn error::Error>;
 pub type DistributionResult<T> = std::result::Result<T, DistributionError>;
@@ -11,7 +12,12 @@ pub type JsonResponse = String;
 #[async_trait]
 pub trait DistributionPlatform {
     /// declare a function
-    async fn declare(&mut self, function_name: &str, project_tar: &[u8]) -> DistributionResult<()>;
+    async fn declare(
+        &mut self,
+        function_name: &str,
+        run_id: Uuid,
+        project_tar: &[u8],
+    ) -> DistributionResult<()>;
 
     // dispatch params to a function
     async fn dispatch(
diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 3d7572f0..d64b7d10 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -18,6 +18,7 @@ use crate::distributed_platform::{
 use crate::utils::RELEASE_FLAG;
 use crate::CACHE_PATH;
 use std::io::Write;
+use uuid::Uuid;
 
 const TURBOLIFT_K8S_NAMESPACE: &str = "default";
 type ImageTag = String;
@@ -68,14 +69,19 @@ impl Default for K8s {
     }
 }
 
-fn function_to_app_name(function_name: &str) -> String {
+fn sanitize_function_name(function_name: &str) -> String {
     function_name.to_string().replace("_", "-")
 }
 
 #[async_trait]
 impl DistributionPlatform for K8s {
     #[tracing::instrument(skip(project_tar))]
-    async fn declare(&mut self, function_name: &str, project_tar: &[u8]) -> DistributionResult<()> {
+    async fn declare(
+        &mut self,
+        function_name: &str,
+        run_id: Uuid,
+        project_tar: &[u8],
+    ) -> DistributionResult<()> {
         // connect to cluster. tries in-cluster configuration first, then falls back to kubeconfig file.
         let deployment_client = Client::try_default().compat().await?;
         let deployments: Api<Deployment> =
@@ -84,7 +90,11 @@ impl DistributionPlatform for K8s {
         let services: Api<Service> = Api::namespaced(service_client, TURBOLIFT_K8S_NAMESPACE);
 
         // generate image & push
-        let app_name = function_to_app_name(function_name);
+        let app_name = format!(
+            "{}-{}",
+            sanitize_function_name(function_name),
+            run_id.as_u128()
+        );
         let container_name = format!("{}-app", app_name);
         let deployment_name = format!("{}-deployment", app_name);
         let service_name = format!("{}-service", app_name);
@@ -163,7 +173,7 @@ impl DistributionPlatform for K8s {
 
         // make ingress pointing to service
         let ingress = serde_json::json!({
-            "apiVersion": "networking.k8s.io/v1beta1",
+            "apiVersion": "networking.k8s.io/v1",
             "kind": "Ingress",
             "metadata": {
                 "name": ingress_name
diff --git a/turbolift_internals/src/lib.rs b/turbolift_internals/src/lib.rs
index b8ccde96..74b697f1 100644
--- a/turbolift_internals/src/lib.rs
+++ b/turbolift_internals/src/lib.rs
@@ -9,6 +9,7 @@ pub mod kubernetes;
 pub mod local_queue;
 pub mod utils;
 pub use serde_json;
+pub use uuid;
 
 lazy_static! {
     /// CACHE_PATH is the directory where turbolift stores derived projects,
diff --git a/turbolift_internals/src/local_queue.rs b/turbolift_internals/src/local_queue.rs
index 274960ee..e5cb5150 100644
--- a/turbolift_internals/src/local_queue.rs
+++ b/turbolift_internals/src/local_queue.rs
@@ -15,6 +15,7 @@ use crate::distributed_platform::{
 };
 use crate::extract_function::decompress_proj_src;
 use crate::CACHE_PATH;
+use uuid::Uuid;
 
 type AddressAndPort = Url;
 type FunctionName = String;
@@ -37,15 +38,23 @@ impl LocalQueue {
 impl DistributionPlatform for LocalQueue {
     /// declare a function. Runs once.
     #[tracing::instrument(skip(project_tar))]
-    async fn declare(&mut self, function_name: &str, project_tar: &[u8]) -> DistributionResult<()> {
+    async fn declare(
+        &mut self,
+        function_name: &str,
+        run_id: Uuid,
+        project_tar: &[u8],
+    ) -> DistributionResult<()> {
         let relative_build_dir = Path::new(".")
             .join(".turbolift")
             .join(".worker_build_cache");
         fs::create_dir_all(&relative_build_dir)?;
         let build_dir = relative_build_dir.canonicalize()?;
         decompress_proj_src(project_tar, &build_dir).unwrap();
-        let function_executable =
-            Path::new(CACHE_PATH.as_os_str()).join(function_name.to_string() + "_server");
+        let function_executable = Path::new(CACHE_PATH.as_os_str()).join(format!(
+            "{}_{}_server",
+            function_name.to_string(),
+            run_id.as_u128()
+        ));
         make_executable(&build_dir.join(function_name), Some(&function_executable))?;
         self.fn_name_to_binary_path
             .insert(function_name.to_string(), function_executable);
diff --git a/turbolift_macros/src/lib.rs b/turbolift_macros/src/lib.rs
index e4fda689..2af0882b 100644
--- a/turbolift_macros/src/lib.rs
+++ b/turbolift_macros/src/lib.rs
@@ -186,6 +186,7 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
             use turbolift::distributed_platform::DistributionPlatform;
             use turbolift::DistributionResult;
             use turbolift::tokio_compat_02::FutureExt;
+            use turbolift::uuid::Uuid;
 
             turbolift::tracing::info!("in original target function");
 
@@ -196,7 +197,7 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
             if !platform.has_declared(#original_target_function_name) {
                 println!("launching declare");
                 platform
-                    .declare(#original_target_function_name, #project_source_binary)
+                    .declare(#original_target_function_name, Uuid::new_v4(), #project_source_binary)
                     .compat()
                     .await?;
                 println!("declare completed");

From 94cc7c7f8c191ac55401b6097ef10816ee97a07d Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sat, 5 Jun 2021 23:52:56 +0200
Subject: [PATCH 105/145] use networking.k8s.io/v1

---
 turbolift_internals/src/kubernetes.rs | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index d64b7d10..bc59b0c3 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -186,17 +186,13 @@ impl DistributionPlatform for K8s {
                                 {
                                     "path": format!("/{}", app_name),
                                     "pathType": "Prefix",
-                                    // "backend": {
-                                    //     "service" : {
-                                    //         "name": app_name,
-                                    //         "port": {
-                                    //             "number": SERVICE_PORT
-                                    //         }
-                                    //     }
-                                    // }
                                     "backend": {
-                                        "serviceName": service_name,
-                                        "servicePort": SERVICE_PORT
+                                        "service" : {
+                                            "name": app_name,
+                                            "port": {
+                                                "number": SERVICE_PORT
+                                            }
+                                        }
                                     }
                                 }
                             ]

From c08556b95e312a080887622440d603dba6badda8 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sun, 6 Jun 2021 00:02:45 +0200
Subject: [PATCH 106/145] extract ignored directories into a constant

---
 turbolift_internals/src/extract_function.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/turbolift_internals/src/extract_function.rs b/turbolift_internals/src/extract_function.rs
index 2d927e59..e9cc15e2 100644
--- a/turbolift_internals/src/extract_function.rs
+++ b/turbolift_internals/src/extract_function.rs
@@ -16,6 +16,8 @@ type TypedParams = syn::punctuated::Punctuated<syn::FnArg, syn::Token![,]>;
 type UntypedParams = syn::punctuated::Punctuated<Box<syn::Pat>, syn::Token![,]>;
 type ParamTypes = syn::punctuated::Punctuated<Box<syn::Type>, syn::Token![,]>;
 
+const IGNORED_DIRECTORIES: [&str; 3] = ["target", ".git", ".turbolift"];
+
 #[tracing::instrument]
 pub fn get_fn_item(function: TokenStream) -> syn::ItemFn {
     match syn::parse2(function).unwrap() {
@@ -180,7 +182,7 @@ pub fn make_compressed_proj_src(dir: &Path) -> Vec<u8> {
     while !entries.is_empty() {
         let (entry_parent, entry) = entries.pop_front().unwrap();
         if entry.metadata().unwrap().is_dir()
-            && (["target", ".git"] // todo could there be cases where removing .git messes up a dependency?
+            && (IGNORED_DIRECTORIES // todo could there be cases where removing .git messes up a dependency?
                 .contains(&entry.file_name().to_str().unwrap_or("")))
         {
             // ignore target and .git repository
@@ -189,9 +191,7 @@ pub fn make_compressed_proj_src(dir: &Path) -> Vec<u8> {
             if entry.path().is_dir() {
                 // ^ bug: the metadata on symlinks sometimes say that they are not directories,
                 // so we have to metadata.is_dir() || (metadata.file_type().is_symlink() && entry.path().is_dir() )
-                if Some("target") == entry.file_name().to_str()
-                    || Some(".turbolift") == entry.file_name().to_str()
-                {
+                if IGNORED_DIRECTORIES.contains(&entry.file_name().to_str().unwrap_or("")) {
                     // don't include any target or .turbolift directories
                 } else {
                     archive

From da9a9e9af3a225749994a91360d039e1e89d3fef Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sun, 6 Jun 2021 09:41:34 +0200
Subject: [PATCH 107/145] try to do multi-stage builds - breaks support for
 targeting multiple architectures at once - breaks support for accessing
 assets bundled in source code directories - makes binaries exponentially
 smaller (GB to MB) - hardcoded to x86 for now

---
 turbolift_internals/src/kubernetes.rs | 46 +++++++++++++++++++--------
 1 file changed, 33 insertions(+), 13 deletions(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index bc59b0c3..8a690762 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -26,6 +26,9 @@ type ImageTag = String;
 pub const CONTAINER_PORT: i32 = 5678;
 pub const SERVICE_PORT: i32 = 5678;
 pub const EXTERNAL_PORT: i32 = 80;
+pub const TARGET_ARCHITECTURE: Option<&str> = Some("x86_64-unknown-linux-musl");
+// ^ todo: let the user set this to enable multi-step builds, or by default ship the source
+// code and run with cargo so that the build can be compiled on the fly.
 
 /// `K8s` is the interface for turning rust functions into autoscaling microservices
 /// using turbolift. It requires docker and kubernetes / kubectl to already be setup on the
@@ -324,7 +327,7 @@ fn make_image(app_name: &str, project_tar: &[u8]) -> anyhow::Result<ImageTag> {
     let tar_file_name = "source.tar";
     let tar_path = build_dir_canonical.join(tar_file_name);
     let docker_file = format!(
-        "FROM ubuntu:latest
+        "FROM ubuntu:latest as builder
 # set timezone (otherwise tzinfo stops dep installation with prompt for time zone)
 ENV TZ=Etc/UTC
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
@@ -337,25 +340,42 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --de
 ENV PATH=/root/.cargo/bin:$PATH
 
 # copy tar file
-COPY {} {}
+COPY {tar_file_name} {tar_file_name}
 
 # unpack tar
-RUN cat {} | tar xvf -
+RUN cat {tar_file_name} | tar xvf -
 
 # enter into unpacked source directory
-WORKDIR {}
+WORKDIR {app_name}
 
 # build and run project
 ENV RUSTFLAGS='--cfg procmacro2_semver_exempt'
-RUN cargo build{}
-CMD cargo run{} 0.0.0.0:{}",
-        tar_file_name,
-        tar_file_name,
-        tar_file_name,
-        app_name,
-        RELEASE_FLAG,
-        RELEASE_FLAG,
-        CONTAINER_PORT
+{compilation_scheme}",
+        app_name=app_name,
+        tar_file_name=tar_file_name,
+        compilation_scheme={
+            if let Some(architecture) = TARGET_ARCHITECTURE {
+                format!("RUN cargo build{release_flag}
+# install the project binary with the given architecture.
+RUN cargo install --target {architecture} --path .
+
+# copy the binary from the builder, leaving the build environment.
+FROM scratch
+COPY --from=builder /usr/local/cargo/bin/{app_name} .
+CMD [\"./{app_name}\", \"0.0.0.0:{container_port}\"]",
+                 architecture=architecture,
+                 release_flag=RELEASE_FLAG,
+                 app_name=app_name,
+                 container_port=CONTAINER_PORT
+                )
+            } else {
+                format!(
+                    "CMD cargo run{release_flag} -- 0.0.0.0:{container_port}",
+                    release_flag=RELEASE_FLAG,
+                    container_port=CONTAINER_PORT
+                )
+            }
+        }
     );
     std::fs::write(&dockerfile_path, docker_file)?;
     std::fs::write(&tar_path, project_tar)?;

From 9fcba7b8255889ffba16a7b2697ff8e62eeebd8f Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sun, 6 Jun 2021 09:55:44 +0200
Subject: [PATCH 108/145] use app name when we need a unique identifier and use
 function name within image environments

---
 turbolift_internals/src/kubernetes.rs | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 8a690762..3bdf37a0 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -102,7 +102,7 @@ impl DistributionPlatform for K8s {
         let deployment_name = format!("{}-deployment", app_name);
         let service_name = format!("{}-service", app_name);
         let ingress_name = format!("{}-ingress", app_name);
-        let tag_in_reg = make_image(&app_name, project_tar)?;
+        let tag_in_reg = make_image(&app_name, function_name, project_tar)?;
 
         println!("image made. making deployment and service names.");
         println!("made service_name");
@@ -314,7 +314,7 @@ lazy_static! {
 }
 
 #[tracing::instrument(skip(project_tar))]
-fn make_image(app_name: &str, project_tar: &[u8]) -> anyhow::Result<ImageTag> {
+fn make_image(app_name: &str, function_name: &str, project_tar: &[u8]) -> anyhow::Result<ImageTag> {
     // todo: we should add some random stuff to the function_name to avoid collisions and figure
     // out when to overwrite vs not.
 
@@ -346,12 +346,12 @@ COPY {tar_file_name} {tar_file_name}
 RUN cat {tar_file_name} | tar xvf -
 
 # enter into unpacked source directory
-WORKDIR {app_name}
+WORKDIR {function_name}
 
 # build and run project
 ENV RUSTFLAGS='--cfg procmacro2_semver_exempt'
 {compilation_scheme}",
-        app_name=app_name,
+        function_name=function_name,
         tar_file_name=tar_file_name,
         compilation_scheme={
             if let Some(architecture) = TARGET_ARCHITECTURE {
@@ -361,11 +361,11 @@ RUN cargo install --target {architecture} --path .
 
 # copy the binary from the builder, leaving the build environment.
 FROM scratch
-COPY --from=builder /usr/local/cargo/bin/{app_name} .
-CMD [\"./{app_name}\", \"0.0.0.0:{container_port}\"]",
+COPY --from=builder /usr/local/cargo/bin/{function_name} .
+CMD [\"./{function_name}\", \"0.0.0.0:{container_port}\"]",
                  architecture=architecture,
                  release_flag=RELEASE_FLAG,
-                 app_name=app_name,
+                 function_name=function_name,
                  container_port=CONTAINER_PORT
                 )
             } else {
@@ -379,7 +379,7 @@ CMD [\"./{app_name}\", \"0.0.0.0:{container_port}\"]",
     );
     std::fs::write(&dockerfile_path, docker_file)?;
     std::fs::write(&tar_path, project_tar)?;
-    let unique_tag = format!("{}:turbolift", app_name); // todo find better unique tag
+    let unique_tag = format!("{}:turbolift", app_name);
 
     let result = (|| {
         // build image

From aab557b89d99f1b2881380a61609bbdd7f56e11c Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sun, 6 Jun 2021 10:12:44 +0200
Subject: [PATCH 109/145] install compiler tools for targeting a specific
 architecture should only be necessary when a target architecture is set

---
 turbolift_internals/src/kubernetes.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 3bdf37a0..e28ac78e 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -348,7 +348,7 @@ RUN cat {tar_file_name} | tar xvf -
 # enter into unpacked source directory
 WORKDIR {function_name}
 
-# build and run project
+# build and run according to compilation scheme
 ENV RUSTFLAGS='--cfg procmacro2_semver_exempt'
 {compilation_scheme}",
         function_name=function_name,
@@ -357,6 +357,7 @@ ENV RUSTFLAGS='--cfg procmacro2_semver_exempt'
             if let Some(architecture) = TARGET_ARCHITECTURE {
                 format!("RUN cargo build{release_flag}
 # install the project binary with the given architecture.
+RUN rustup target add {architecture}
 RUN cargo install --target {architecture} --path .
 
 # copy the binary from the builder, leaving the build environment.

From cf98731f236cd6e8fc14b076ac9dbb73109d6b9f Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sun, 6 Jun 2021 10:59:05 +0200
Subject: [PATCH 110/145] remove extraneous build

---
 turbolift_internals/src/kubernetes.rs | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index e28ac78e..9fc42c76 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -355,10 +355,9 @@ ENV RUSTFLAGS='--cfg procmacro2_semver_exempt'
         tar_file_name=tar_file_name,
         compilation_scheme={
             if let Some(architecture) = TARGET_ARCHITECTURE {
-                format!("RUN cargo build{release_flag}
-# install the project binary with the given architecture.
+                format!("# install the project binary with the given architecture.
 RUN rustup target add {architecture}
-RUN cargo install --target {architecture} --path .
+RUN cargo install{release_flag} --target {architecture} --path .
 
 # copy the binary from the builder, leaving the build environment.
 FROM scratch

From c67012601c1c10f2be3f4c2c87ce6c5e05858aca Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sun, 6 Jun 2021 11:06:39 +0200
Subject: [PATCH 111/145] don't follow target architecture path

Unfortunately, it looks like the special unstable code we need to be able to
work with Spans while extracting service source code is not compatible with
the musl target. This means that we can't (for now) use multi-stage builds.
---
 turbolift_internals/src/kubernetes.rs | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 9fc42c76..fe61b081 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -26,9 +26,11 @@ type ImageTag = String;
 pub const CONTAINER_PORT: i32 = 5678;
 pub const SERVICE_PORT: i32 = 5678;
 pub const EXTERNAL_PORT: i32 = 80;
-pub const TARGET_ARCHITECTURE: Option<&str> = Some("x86_64-unknown-linux-musl");
-// ^ todo: let the user set this to enable multi-step builds, or by default ship the source
-// code and run with cargo so that the build can be compiled on the fly.
+pub const TARGET_ARCHITECTURE: Option<&str> = None;
+// ^ todo: we want the user to be able to specify something like `Some("x86_64-unknown-linux-musl")`
+//   during config, but right now that doesn't work because we are relying on super unstable
+//   span features to extract functions into services. When we can enable statically linked
+//   targets, we can use the multi-stage build path and significantly reduce the size.
 
 /// `K8s` is the interface for turning rust functions into autoscaling microservices
 /// using turbolift. It requires docker and kubernetes / kubectl to already be setup on the

From 580dc1c77b50752400b622269ed8689372e40bd3 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sun, 6 Jun 2021 12:44:47 +0200
Subject: [PATCH 112/145] pass debug flag to install

cargo install takes a debug flag as an argument
instead of release (it does release as default).
---
 turbolift_internals/src/kubernetes.rs | 6 +++---
 turbolift_internals/src/utils.rs      | 8 ++++++++
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index fe61b081..d8e1a9a6 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -15,7 +15,7 @@ use url::Url;
 use crate::distributed_platform::{
     ArgsString, DistributionPlatform, DistributionResult, JsonResponse,
 };
-use crate::utils::RELEASE_FLAG;
+use crate::utils::{DEBUG_FLAG, RELEASE_FLAG};
 use crate::CACHE_PATH;
 use std::io::Write;
 use uuid::Uuid;
@@ -359,14 +359,14 @@ ENV RUSTFLAGS='--cfg procmacro2_semver_exempt'
             if let Some(architecture) = TARGET_ARCHITECTURE {
                 format!("# install the project binary with the given architecture.
 RUN rustup target add {architecture}
-RUN cargo install{release_flag} --target {architecture} --path .
+RUN cargo install{debug_flag} --target {architecture} --path .
 
 # copy the binary from the builder, leaving the build environment.
 FROM scratch
 COPY --from=builder /usr/local/cargo/bin/{function_name} .
 CMD [\"./{function_name}\", \"0.0.0.0:{container_port}\"]",
                  architecture=architecture,
-                 release_flag=RELEASE_FLAG,
+                 debug_flag=DEBUG_FLAG,
                  function_name=function_name,
                  container_port=CONTAINER_PORT
                 )
diff --git a/turbolift_internals/src/utils.rs b/turbolift_internals/src/utils.rs
index b288ebef..85d49691 100644
--- a/turbolift_internals/src/utils.rs
+++ b/turbolift_internals/src/utils.rs
@@ -18,3 +18,11 @@ pub const RELEASE_FLAG: &str = {
         ""
     }
 };
+
+pub const DEBUG_FLAG: &str = {
+    if IS_RELEASE {
+        ""
+    } else {
+        " --debug"
+    }
+};

From 912d7bd128cb9fbdc27b587af85a582666d49b50 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sun, 6 Jun 2021 12:46:02 +0200
Subject: [PATCH 113/145] point ingress to service

---
 turbolift_internals/src/kubernetes.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index d8e1a9a6..e1e0a129 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -193,7 +193,7 @@ impl DistributionPlatform for K8s {
                                     "pathType": "Prefix",
                                     "backend": {
                                         "service" : {
-                                            "name": app_name,
+                                            "name": service_name,
                                             "port": {
                                                 "number": SERVICE_PORT
                                             }

From 7df3a6e9c3aa33aa7647a3c2caf61b911195993b Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sun, 6 Jun 2021 20:46:26 +0200
Subject: [PATCH 114/145] add debug prints

---
 examples/kubernetes_example/src/main.rs | 1 +
 turbolift_macros/src/lib.rs             | 6 ++++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/examples/kubernetes_example/src/main.rs b/examples/kubernetes_example/src/main.rs
index aad94941..8d6c2dd9 100644
--- a/examples/kubernetes_example/src/main.rs
+++ b/examples/kubernetes_example/src/main.rs
@@ -17,6 +17,7 @@ lazy_static! {
 
 #[on(K8S)]
 fn square(u: u64) -> u64 {
+    println!("🟩");
     u * u
 }
 
diff --git a/turbolift_macros/src/lib.rs b/turbolift_macros/src/lib.rs
index 2af0882b..4d0a1276 100644
--- a/turbolift_macros/src/lib.rs
+++ b/turbolift_macros/src/lib.rs
@@ -59,6 +59,7 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
         #[get(#wrapper_route)]
         #[turbolift::tracing::instrument]
         async fn turbolift_wrapper(web::Path((#untyped_params_tokens)): web::Path<(#param_types)>) -> Result<HttpResponse> {
+            println!("in the wappa");
             Ok(
                 HttpResponse::Ok()
                     .json(#function_name(#untyped_params_tokens))
@@ -188,11 +189,11 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
             use turbolift::tokio_compat_02::FutureExt;
             use turbolift::uuid::Uuid;
 
-            turbolift::tracing::info!("in original target function");
+            println!("in original target function");
 
             let mut platform = #distribution_platform.lock().await;
 
-            turbolift::tracing::info!("platform acquired");
+            println!("platform acquired");
 
             if !platform.has_declared(#original_target_function_name) {
                 println!("launching declare");
@@ -242,6 +243,7 @@ pub fn on(_distribution_platform: TokenStream, function_: TokenStream) -> TokenS
         #[turbolift::tracing::instrument]
         async fn #original_target_function_ident(#typed_params) -> turbolift::DistributionResult<#output_type> {
             #wrapped_original_function
+            println!("wuuuh in funcypoo: {}", wrapped_function(#untyped_params));
             Ok(wrapped_function(#untyped_params))
         }
     };

From 4b55ef317396c16a51d26a095d7c0ea1edd66f23 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sun, 6 Jun 2021 20:48:47 +0200
Subject: [PATCH 115/145] remove unused service

---
 turbolift_macros/src/lib.rs | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/turbolift_macros/src/lib.rs b/turbolift_macros/src/lib.rs
index 4d0a1276..0a065b68 100644
--- a/turbolift_macros/src/lib.rs
+++ b/turbolift_macros/src/lib.rs
@@ -66,10 +66,6 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
             )
         }
 
-        async fn return_path(req: HttpRequest) -> impl Responder {
-            HttpResponse::Ok().body(req.uri().to_string())
-        }
-
         #[actix_web::main]
         #[turbolift::tracing::instrument]
         async fn main() -> std::io::Result<()> {

From 8fa5483cd990e09c55cafdd665283e03b0e7ec29 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Mon, 7 Jun 2021 21:13:51 +0200
Subject: [PATCH 116/145] change path to services to line up with the path
 called by the client

---
 examples/kubernetes_example/run_tests.sh    | 8 ++++++++
 turbolift_internals/src/extract_function.rs | 2 +-
 turbolift_macros/src/lib.rs                 | 5 +----
 3 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/examples/kubernetes_example/run_tests.sh b/examples/kubernetes_example/run_tests.sh
index 0f9cb95c..000d544d 100644
--- a/examples/kubernetes_example/run_tests.sh
+++ b/examples/kubernetes_example/run_tests.sh
@@ -14,6 +14,10 @@ printf "\n📍 running non-distributed tests\n"
 . ./run_non_distributed_tests.sh
 echo "non-distributed tests completed."
 
+printf "\n🚽 deleting target folder to save space in CI\n"
+rm -r ./target
+echo "target folder deleted"
+
 printf "\n👷 setting up cluster with custom ingress-compatible config\n"
 cat <<EOF | kind create cluster --config=-
 kind: Cluster
@@ -52,6 +56,10 @@ printf "\n🤸‍ run distributed tests\n"
 . ./run_distributed_tests.sh
 echo "🤸 distributed tests completed."
 
+printf "\n🚽 deleting target folder to save space in CI\n"
+rm -r ./target
+echo "target folder deleted"
+
 printf "\n📍 re-run non-distributed tests\n"
 . ./run_non_distributed_tests.sh
 echo "📍 non-distributed tests completed."
diff --git a/turbolift_internals/src/extract_function.rs b/turbolift_internals/src/extract_function.rs
index e9cc15e2..5dffd27b 100644
--- a/turbolift_internals/src/extract_function.rs
+++ b/turbolift_internals/src/extract_function.rs
@@ -94,7 +94,7 @@ pub fn to_path_params(untyped_params: UntypedParams) -> String {
         .map(|pat| open_bracket.to_string() + &pat.into_token_stream().to_string() + &close_bracket)
         .collect();
 
-    path_format.join("/")
+    format!("/{}", path_format.join("/"))
 }
 
 #[tracing::instrument]
diff --git a/turbolift_macros/src/lib.rs b/turbolift_macros/src/lib.rs
index 0a065b68..618a58fd 100644
--- a/turbolift_macros/src/lib.rs
+++ b/turbolift_macros/src/lib.rs
@@ -32,8 +32,6 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
     let untyped_params = extract_function::to_untyped_params(typed_params.clone());
     let untyped_params_tokens = untyped_params.to_token_stream();
     let params_as_path = extract_function::to_path_params(untyped_params.clone());
-    let wrapper_route = format!("/{}/{}", &original_target_function_name, &params_as_path);
-    // ^ todo: make this unique even if multiple functions with the same name are distributed at the same time
     let param_types = extract_function::to_param_types(typed_params.clone());
     let params_vec = extract_function::params_json_vec(untyped_params.clone());
     let result_type = extract_function::get_result_type(&signature.output);
@@ -56,7 +54,7 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
         #dummy_function
         #target_function
 
-        #[get(#wrapper_route)]
+        #[get(#params_as_path)]
         #[turbolift::tracing::instrument]
         async fn turbolift_wrapper(web::Path((#untyped_params_tokens)): web::Path<(#param_types)>) -> Result<HttpResponse> {
             println!("in the wappa");
@@ -239,7 +237,6 @@ pub fn on(_distribution_platform: TokenStream, function_: TokenStream) -> TokenS
         #[turbolift::tracing::instrument]
         async fn #original_target_function_ident(#typed_params) -> turbolift::DistributionResult<#output_type> {
             #wrapped_original_function
-            println!("wuuuh in funcypoo: {}", wrapped_function(#untyped_params));
             Ok(wrapped_function(#untyped_params))
         }
     };

From 8aea3261206eb33061b43284f70180dc5d5488d5 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Mon, 7 Jun 2021 21:21:37 +0200
Subject: [PATCH 117/145] delete debug print

---
 examples/kubernetes_example/src/main.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/examples/kubernetes_example/src/main.rs b/examples/kubernetes_example/src/main.rs
index 8d6c2dd9..aad94941 100644
--- a/examples/kubernetes_example/src/main.rs
+++ b/examples/kubernetes_example/src/main.rs
@@ -17,7 +17,6 @@ lazy_static! {
 
 #[on(K8S)]
 fn square(u: u64) -> u64 {
-    println!("🟩");
     u * u
 }
 

From d3927396b67d106dc204950dceb521a51747a034 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Mon, 7 Jun 2021 21:32:35 +0200
Subject: [PATCH 118/145] re-add service name into path

---
 turbolift_internals/src/extract_function.rs | 2 +-
 turbolift_internals/src/kubernetes.rs       | 7 +++++--
 turbolift_macros/src/lib.rs                 | 3 ++-
 3 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/turbolift_internals/src/extract_function.rs b/turbolift_internals/src/extract_function.rs
index 5dffd27b..e9cc15e2 100644
--- a/turbolift_internals/src/extract_function.rs
+++ b/turbolift_internals/src/extract_function.rs
@@ -94,7 +94,7 @@ pub fn to_path_params(untyped_params: UntypedParams) -> String {
         .map(|pat| open_bracket.to_string() + &pat.into_token_stream().to_string() + &close_bracket)
         .collect();
 
-    format!("/{}", path_format.join("/"))
+    path_format.join("/")
 }
 
 #[tracing::instrument]
diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index e1e0a129..af58e0d2 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -189,7 +189,7 @@ impl DistributionPlatform for K8s {
                         "http": {
                             "paths": [
                                 {
-                                    "path": format!("/{}", app_name),
+                                    "path": format!("/{}/{}", app_name, function_name),
                                     "pathType": "Prefix",
                                     "backend": {
                                         "service" : {
@@ -245,7 +245,10 @@ impl DistributionPlatform for K8s {
         //     .next()
         //     .expect("no node port assigned to service");
         // let service_ip = format!("http://{}", node_ip, node_port);
-        let service_ip = format!("http://localhost:{}/{}/", EXTERNAL_PORT, app_name);
+        let service_ip = format!(
+            "http://localhost:{}/{}/{}/",
+            EXTERNAL_PORT, app_name, function_name
+        );
         println!("generated service_ip: {}", service_ip.as_str());
 
         // todo make sure that the pod and service were correctly started before returning
diff --git a/turbolift_macros/src/lib.rs b/turbolift_macros/src/lib.rs
index 618a58fd..f66324bb 100644
--- a/turbolift_macros/src/lib.rs
+++ b/turbolift_macros/src/lib.rs
@@ -32,6 +32,7 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
     let untyped_params = extract_function::to_untyped_params(typed_params.clone());
     let untyped_params_tokens = untyped_params.to_token_stream();
     let params_as_path = extract_function::to_path_params(untyped_params.clone());
+    let wrapper_route = format!("/{}/{}", &original_target_function_name, &params_as_path);
     let param_types = extract_function::to_param_types(typed_params.clone());
     let params_vec = extract_function::params_json_vec(untyped_params.clone());
     let result_type = extract_function::get_result_type(&signature.output);
@@ -54,7 +55,7 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
         #dummy_function
         #target_function
 
-        #[get(#params_as_path)]
+        #[get(#wrapper_route)]
         #[turbolift::tracing::instrument]
         async fn turbolift_wrapper(web::Path((#untyped_params_tokens)): web::Path<(#param_types)>) -> Result<HttpResponse> {
             println!("in the wappa");

From 785678f26f376175a86443b7ea5d4420893de130 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Wed, 9 Jun 2021 08:07:22 +0200
Subject: [PATCH 119/145] add additional debug statements

---
 turbolift_internals/src/kubernetes.rs  | 2 +-
 turbolift_internals/src/local_queue.rs | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index af58e0d2..d292eedc 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -295,7 +295,7 @@ impl DistributionPlatform for K8s {
         let args = format!("./{}", params);
         let query_url = service_base_url.join(&args)?;
         tracing::info!(url = query_url.as_str(), "sending dispatch request");
-        println!("sending dispatch request");
+        println!("sending dispatch request to {}", query_url.as_str());
         Ok(self
             .request_client
             .get(query_url)
diff --git a/turbolift_internals/src/local_queue.rs b/turbolift_internals/src/local_queue.rs
index e5cb5150..bc8c503f 100644
--- a/turbolift_internals/src/local_queue.rs
+++ b/turbolift_internals/src/local_queue.rs
@@ -103,6 +103,7 @@ impl DistributionPlatform for LocalQueue {
         let query_url = address_and_port.join(&prefixed_params)?;
 
         tracing::info!("sending dispatch request");
+        println!("sending dispatch request to {}", query_url.as_str());
         let resp = Ok(self
             .request_client
             .get(query_url)

From e326048e1a568e5630485b8077d093d9dbf68ab0 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Wed, 9 Jun 2021 08:49:30 +0200
Subject: [PATCH 120/145] fix in-cluster url

---
 turbolift_internals/src/kubernetes.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index d292eedc..c45b5eae 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -189,7 +189,7 @@ impl DistributionPlatform for K8s {
                         "http": {
                             "paths": [
                                 {
-                                    "path": format!("/{}/{}", app_name, function_name),
+                                    "path": format!("/{}", app_name),
                                     "pathType": "Prefix",
                                     "backend": {
                                         "service" : {
@@ -278,7 +278,6 @@ impl DistributionPlatform for K8s {
 
         self.fn_names_to_services
             .insert(function_name.to_string(), Url::from_str(&service_ip)?);
-        // todo handle deleting the relevant service and deployment for each distributed function.
 
         println!("returning from declare");
         Ok(())

From b246746c8fd3370fd067431e0732d7eb0fff117a Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Fri, 11 Jun 2021 23:14:57 +0200
Subject: [PATCH 121/145] reformat service url to pass run id as a parameter

---
 .github/workflows/examples.yml         |  2 ++
 turbolift_internals/src/kubernetes.rs  | 13 +++----
 turbolift_internals/src/local_queue.rs |  2 +-
 turbolift_macros/Cargo.toml            |  1 +
 turbolift_macros/src/lib.rs            | 48 +++++++++++++++++++++-----
 5 files changed, 49 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml
index cb235713..1d47d65f 100644
--- a/.github/workflows/examples.yml
+++ b/.github/workflows/examples.yml
@@ -21,6 +21,8 @@ jobs:
         with:
           path: './turbolift'
       - uses: engineerd/setup-kind@v0.5.0
+        with:
+          version: "v0.11.0"
       - name: install rustup and rust nightly
         run: |
             curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly-2020-09-28
diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index c45b5eae..1e4a4253 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -95,11 +95,7 @@ impl DistributionPlatform for K8s {
         let services: Api<Service> = Api::namespaced(service_client, TURBOLIFT_K8S_NAMESPACE);
 
         // generate image & push
-        let app_name = format!(
-            "{}-{}",
-            sanitize_function_name(function_name),
-            run_id.as_u128()
-        );
+        let app_name = format!("{}-{}", sanitize_function_name(function_name), run_id);
         let container_name = format!("{}-app", app_name);
         let deployment_name = format!("{}-deployment", app_name);
         let service_name = format!("{}-service", app_name);
@@ -189,7 +185,7 @@ impl DistributionPlatform for K8s {
                         "http": {
                             "paths": [
                                 {
-                                    "path": format!("/{}", app_name),
+                                    "path": format!("/{}/{}", function_name, run_id),
                                     "pathType": "Prefix",
                                     "backend": {
                                         "service" : {
@@ -247,7 +243,7 @@ impl DistributionPlatform for K8s {
         // let service_ip = format!("http://{}", node_ip, node_port);
         let service_ip = format!(
             "http://localhost:{}/{}/{}/",
-            EXTERNAL_PORT, app_name, function_name
+            EXTERNAL_PORT, function_name, run_id
         );
         println!("generated service_ip: {}", service_ip.as_str());
 
@@ -374,7 +370,8 @@ CMD [\"./{function_name}\", \"0.0.0.0:{container_port}\"]",
                 )
             } else {
                 format!(
-                    "CMD cargo run{release_flag} -- 0.0.0.0:{container_port}",
+                    "RUN cargo build{release_flag}
+                     CMD cargo run{release_flag} -- 0.0.0.0:{container_port}",
                     release_flag=RELEASE_FLAG,
                     container_port=CONTAINER_PORT
                 )
diff --git a/turbolift_internals/src/local_queue.rs b/turbolift_internals/src/local_queue.rs
index bc8c503f..98721d0f 100644
--- a/turbolift_internals/src/local_queue.rs
+++ b/turbolift_internals/src/local_queue.rs
@@ -99,7 +99,7 @@ impl DistributionPlatform for LocalQueue {
         };
 
         // request from server
-        let prefixed_params = "./".to_string() + function_name + "/" + &params;
+        let prefixed_params = "./".to_string() + function_name + "/empty-uuid/" + &params;
         let query_url = address_and_port.join(&prefixed_params)?;
 
         tracing::info!("sending dispatch request");
diff --git a/turbolift_macros/Cargo.toml b/turbolift_macros/Cargo.toml
index f5404c5c..c4016918 100644
--- a/turbolift_macros/Cargo.toml
+++ b/turbolift_macros/Cargo.toml
@@ -21,3 +21,4 @@ futures = "0.3"
 cached = "0.19"
 tracing = {version="0.1", features=["attributes"]}
 tracing-futures = "0.2.4"
+syn = { version = "1", features=["full"] }
diff --git a/turbolift_macros/src/lib.rs b/turbolift_macros/src/lib.rs
index f66324bb..381698c4 100644
--- a/turbolift_macros/src/lib.rs
+++ b/turbolift_macros/src/lib.rs
@@ -1,5 +1,5 @@
 use proc_macro::TokenStream;
-use proc_macro2::TokenStream as TokenStream2;
+use proc_macro2::{Ident, Span, TokenStream as TokenStream2};
 use quote::quote as q;
 
 use turbolift_internals::extract_function;
@@ -15,6 +15,8 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
 
     use turbolift_internals::{build_project, CACHE_PATH};
 
+    const RUN_ID_NAME: &str = "_turbolift_run_id";
+
     // convert proc_macro::TokenStream to proc_macro2::TokenStream
     let distribution_platform = TokenStream2::from(distribution_platform_);
     let function = TokenStream2::from(function_);
@@ -30,10 +32,34 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
     let function_name_string = function_name.to_string();
     let typed_params = signature.inputs;
     let untyped_params = extract_function::to_untyped_params(typed_params.clone());
+    let mut untyped_params_with_run_id = untyped_params.clone();
+    // we need to prepend a variable for the run id, since it's in the URL.
+    untyped_params_with_run_id.insert(
+        0,
+        Box::new(syn::Pat::Ident(syn::PatIdent {
+            attrs: vec![],
+            by_ref: None,
+            mutability: None,
+            ident: Ident::new(RUN_ID_NAME, Span::call_site()),
+            subpat: None,
+        })),
+    );
+    let untyped_params_tokens_with_run_id = untyped_params_with_run_id.to_token_stream();
     let untyped_params_tokens = untyped_params.to_token_stream();
     let params_as_path = extract_function::to_path_params(untyped_params.clone());
-    let wrapper_route = format!("/{}/{}", &original_target_function_name, &params_as_path);
-    let param_types = extract_function::to_param_types(typed_params.clone());
+    let wrapper_route = format!(
+        "{}/{{{}}}/{}",
+        original_target_function_name, RUN_ID_NAME, &params_as_path
+    );
+    let mut param_types = extract_function::to_param_types(typed_params.clone());
+    // we need to prepend a type for the run id added to the wrapper route
+    param_types.insert(
+        0,
+        Box::new(syn::Type::Verbatim(
+            str::parse::<TokenStream2>("String")
+                .expect("could not parse \"String\" as a tokenstream"),
+        )),
+    );
     let params_vec = extract_function::params_json_vec(untyped_params.clone());
     let result_type = extract_function::get_result_type(&signature.output);
     let dummy_function = extract_function::make_dummy_function(
@@ -57,8 +83,7 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
 
         #[get(#wrapper_route)]
         #[turbolift::tracing::instrument]
-        async fn turbolift_wrapper(web::Path((#untyped_params_tokens)): web::Path<(#param_types)>) -> Result<HttpResponse> {
-            println!("in the wappa");
+        async fn turbolift_wrapper(web::Path((#untyped_params_tokens_with_run_id)): web::Path<(#param_types)>) -> Result<HttpResponse> {
             Ok(
                 HttpResponse::Ok()
                     .json(#function_name(#untyped_params_tokens))
@@ -68,7 +93,7 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
         #[actix_web::main]
         #[turbolift::tracing::instrument]
         async fn main() -> std::io::Result<()> {
-            use actix_web::{App, HttpServer};
+            use actix_web::{App, HttpServer, HttpRequest, web};
 
             let args: Vec<String> = std::env::args().collect();
             let ip_and_port = &args[1];
@@ -79,6 +104,15 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
                         .service(
                             turbolift_wrapper
                         )
+                        .default_service(
+                            web::get()
+                                .to(
+                                    |req: HttpRequest|
+                                        HttpResponse::NotFound().body(
+                                            format!("endpoint not found: {}", req.uri())
+                                        )
+                                )
+                        )
             )
             .bind(ip_and_port)?
             .run()
@@ -220,8 +254,6 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
 #[cfg(not(feature = "distributed"))]
 #[proc_macro_attribute]
 pub fn on(_distribution_platform: TokenStream, function_: TokenStream) -> TokenStream {
-    use proc_macro2::{Ident, Span};
-
     // convert proc_macro::TokenStream to proc_macro2::TokenStream
     let function = TokenStream2::from(function_);
     let mut wrapped_original_function = extract_function::get_fn_item(function);

From 71d324597b8e2ca757d00e5dc0a8b577b73395e8 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sat, 12 Jun 2021 11:02:56 +0200
Subject: [PATCH 122/145] add function to make the container accessible from
 the cluster into the K8s global constructor

---
 examples/kubernetes_example/Cargo.toml  |  1 +
 examples/kubernetes_example/src/main.rs | 20 +++++-
 turbolift_internals/Cargo.toml          |  1 +
 turbolift_internals/src/kubernetes.rs   | 96 ++++++++++---------------
 4 files changed, 58 insertions(+), 60 deletions(-)

diff --git a/examples/kubernetes_example/Cargo.toml b/examples/kubernetes_example/Cargo.toml
index 52cf7bec..3a52ea5e 100644
--- a/examples/kubernetes_example/Cargo.toml
+++ b/examples/kubernetes_example/Cargo.toml
@@ -13,6 +13,7 @@ tokio = {version="1", features=["full"]}
 lazy_static = "1"
 futures = "0.3"
 cute = "0.3"
+anyhow = "1.0.41"
 turbolift = { path="../../" }
 
 # for printing out tracing
diff --git a/examples/kubernetes_example/src/main.rs b/examples/kubernetes_example/src/main.rs
index aad94941..53a0aafc 100644
--- a/examples/kubernetes_example/src/main.rs
+++ b/examples/kubernetes_example/src/main.rs
@@ -11,10 +11,28 @@ use tracing_subscriber;
 use turbolift::kubernetes::K8s;
 use turbolift::on;
 
+/// instantiate the global cluster manager
 lazy_static! {
-    static ref K8S: Mutex<K8s> = Mutex::new(K8s::with_max_replicas(2));
+    static ref K8S: Mutex<K8s> = Mutex::new(K8s::with_deploy_function_and_max_replicas(
+        Box::new(load_container_into_kind),
+        2
+    ));
 }
 
+/// The application writer is responsible for placing
+/// images where your cluster can access them. The
+/// K8s constructor has a parameter which takes
+/// a function that is called after the container is
+/// built, so that the container may be added to a
+/// specific registry or otherwise be made available.
+fn load_container_into_kind(tag: &str) -> anyhow::Result<&str> {
+    std::process::Command::new("kind")
+        .args(format!("load docker-image {}", tag).as_str().split(' '))
+        .status()?;
+    Ok(tag)
+}
+
+/// This is the function we want to distribute!
 #[on(K8S)]
 fn square(u: u64) -> u64 {
     u * u
diff --git a/turbolift_internals/Cargo.toml b/turbolift_internals/Cargo.toml
index 2ad24065..f597bfd6 100644
--- a/turbolift_internals/Cargo.toml
+++ b/turbolift_internals/Cargo.toml
@@ -34,6 +34,7 @@ regex = "1"
 tracing = {version="0.1", features=["attributes"]}
 tracing-futures = "0.2.4"
 uuid = { version="0.8", features=["v4"] }
+derivative = "2.2.0"
 
 # kubernetes-specific requirements
 kube = "0.51.0"
diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 1e4a4253..96fb1242 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -3,6 +3,7 @@ use std::process::{Command, Stdio};
 use std::str::FromStr;
 
 use async_trait::async_trait;
+use derivative::Derivative;
 use k8s_openapi::api::apps::v1::Deployment;
 use k8s_openapi::api::core::v1::Service;
 use kube::api::{Api, PostParams};
@@ -22,6 +23,7 @@ use uuid::Uuid;
 
 const TURBOLIFT_K8S_NAMESPACE: &str = "default";
 type ImageTag = String;
+type DeployContainerFunction = Box<dyn Fn(&str) -> anyhow::Result<&str> + Send + 'static>;
 
 pub const CONTAINER_PORT: i32 = 5678;
 pub const SERVICE_PORT: i32 = 5678;
@@ -32,35 +34,51 @@ pub const TARGET_ARCHITECTURE: Option<&str> = None;
 //   span features to extract functions into services. When we can enable statically linked
 //   targets, we can use the multi-stage build path and significantly reduce the size.
 
+#[derive(Derivative)]
+#[derivative(Debug)]
 /// `K8s` is the interface for turning rust functions into autoscaling microservices
 /// using turbolift. It requires docker and kubernetes / kubectl to already be setup on the
 /// device at runtime.
 ///
-/// Access to the kubernetes cluster must be inferrable from the env variables at runtime.
-#[derive(Debug)]
+/// Access to the kubernetes cluster must be inferrable from the env variables at runtime
+/// per kube-rs's
+/// [try_default()](https://docs.rs/kube/0.56.0/kube/client/struct.Client.html#method.try_default).
 pub struct K8s {
     max_scale_n: u32,
     fn_names_to_services: HashMap<String, Url>,
     request_client: reqwest::Client,
+
+    #[derivative(Debug = "ignore")]
+    /// A function called after the image is built locally via docker. deploy_container
+    /// receives the tag for the local image (accessible in docker) and is responsible
+    /// for making said image accessible to the target cluster. The output of
+    /// deploy_container is the tag that Kubernetes can use to refer to and access the
+    /// image throughout the cluster.
+    ///
+    /// Some examples of how this function can be implemented: uploading the image to
+    /// the cluster's private registry, uploading the image publicly to docker hub
+    /// (if the image is not sensitive), loading the image into KinD in tests.
+    deploy_container: DeployContainerFunction,
 }
 
 impl K8s {
-    /// returns a K8s object that does not perform autoscaling.
-    #[tracing::instrument]
-    pub fn new() -> K8s {
-        K8s::with_max_replicas(1)
-    }
-
     /// returns a K8s object. If max is equal to 1, then autoscaling
     /// is not enabled. Otherwise, autoscale is automatically activated
     /// with cluster defaults and a max number of replicas *per distributed
     /// function* of `max`. Panics if `max` < 1.
-    #[tracing::instrument]
-    pub fn with_max_replicas(max: u32) -> K8s {
+    ///
+    /// The deploy container function is used for making containers accessible
+    /// to the cluster. See [`K8s::deploy_container`].
+    #[tracing::instrument(skip(deploy_container))]
+    pub fn with_deploy_function_and_max_replicas(
+        deploy_container: DeployContainerFunction,
+        max: u32,
+    ) -> K8s {
         if max < 1 {
             panic!("max < 1 while instantiating k8s (value: {})", max)
         }
         K8s {
+            deploy_container,
             max_scale_n: max,
             fn_names_to_services: HashMap::new(),
             request_client: reqwest::Client::new(),
@@ -68,12 +86,6 @@ impl K8s {
     }
 }
 
-impl Default for K8s {
-    fn default() -> Self {
-        K8s::new()
-    }
-}
-
 fn sanitize_function_name(function_name: &str) -> String {
     function_name.to_string().replace("_", "-")
 }
@@ -100,7 +112,7 @@ impl DistributionPlatform for K8s {
         let deployment_name = format!("{}-deployment", app_name);
         let service_name = format!("{}-service", app_name);
         let ingress_name = format!("{}-ingress", app_name);
-        let tag_in_reg = make_image(&app_name, function_name, project_tar)?;
+        let tag_in_reg = make_image(self, &app_name, function_name, project_tar)?;
 
         println!("image made. making deployment and service names.");
         println!("made service_name");
@@ -269,7 +281,7 @@ impl DistributionPlatform for K8s {
         //     }
         // }
 
-        sleep(Duration::from_secs(600)).await;
+        sleep(Duration::from_secs(90)).await;
         // todo implement the check on whether the service is running / pod failed
 
         self.fn_names_to_services
@@ -314,7 +326,12 @@ lazy_static! {
 }
 
 #[tracing::instrument(skip(project_tar))]
-fn make_image(app_name: &str, function_name: &str, project_tar: &[u8]) -> anyhow::Result<ImageTag> {
+fn make_image(
+    k8s: &K8s,
+    app_name: &str,
+    function_name: &str,
+    project_tar: &[u8],
+) -> anyhow::Result<ImageTag> {
     // todo: we should add some random stuff to the function_name to avoid collisions and figure
     // out when to overwrite vs not.
 
@@ -398,37 +415,6 @@ CMD [\"./{function_name}\", \"0.0.0.0:{container_port}\"]",
             return Err(anyhow::anyhow!("docker image build failure"));
         }
 
-        // let image_tag = format!(
-        //     "localhost:{}/{}",
-        //     registry_url.port().unwrap(),
-        //     function_name
-        // );
-
-        println!("made image tag");
-
-        // tag image
-        // let tag_args = format!("image tag {} {}", function_name, function_name);
-        // let tag_result = Command::new("docker")
-        //     .args(tag_args.as_str().split(' '))
-        //     .status()?;
-        // if !tag_result.success() {
-        //     return Err(anyhow::anyhow!("docker image tag failure"));
-        // }
-        // tracing::info!("image tagged");
-
-        // push image to local repo
-        // let image_tag = format!("dominicburkart/{}:latest", function_name.clone());
-        // let push_status = Command::new("docker")
-        //     .arg("push")
-        //     .arg(image_tag.clone())
-        //     .status()?;
-        // tracing::info!("docker push command did not explode");
-        // if !push_status.success() {
-        //     return Err(anyhow::anyhow!("docker image push failure"));
-        // }
-
-        // println!("haha >:D {}", image_tag);
-
         Ok(unique_tag.clone())
     })();
     println!("removing build dir");
@@ -436,15 +422,7 @@ CMD [\"./{function_name}\", \"0.0.0.0:{container_port}\"]",
     std::fs::remove_dir_all(build_dir_canonical)?;
     println!("returning result");
 
-    Command::new("kind")
-        .args(
-            format!("load docker-image {}", unique_tag)
-                .as_str()
-                .split(' '),
-        )
-        .status()?;
-    // todo ^ temp fix while debugging kind
-    result
+    result.and((k8s.deploy_container)(unique_tag.as_str()).map(|s| s.to_string()))
 }
 
 impl Drop for K8s {

From 366183276c8f29a195374dbed7f3ba002d5a84e1 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sat, 12 Jun 2021 11:19:57 +0200
Subject: [PATCH 123/145] refactor kubernetes test suite

---
 .github/workflows/examples.yml                | 53 +++++++++++++++++--
 .../run_distributed_tests.sh                  |  8 ---
 .../run_non_distributed_tests.sh              |  8 ---
 examples/kubernetes_example/run_tests.sh      | 17 +++---
 4 files changed, 55 insertions(+), 31 deletions(-)
 delete mode 100644 examples/kubernetes_example/run_distributed_tests.sh
 delete mode 100644 examples/kubernetes_example/run_non_distributed_tests.sh

diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml
index 1d47d65f..11da9bb7 100644
--- a/.github/workflows/examples.yml
+++ b/.github/workflows/examples.yml
@@ -14,19 +14,64 @@ jobs:
       - name: build local queue example
         run: docker build -f examples/local_queue_example/Dockerfile .
 
-  kubernetes_example:
+  kubernetes_example_local_test:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
         with:
           path: './turbolift'
+      - name: install rustup and rust nightly
+        run: |
+            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly-2020-09-28
+      - name: run tests
+        run: |
+            cd turbolift/examples/kubernetes_example
+            RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo test -- --nocapture
+
+  kubernetes_example_local_run:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          path: './turbolift'
+      - name: install rustup and rust nightly
+        run: |
+          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly-2020-09-28
+      - name: run tests
+        run: |
+          cd turbolift/examples/kubernetes_example
+          RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run
+
+  kubernetes_example_distributed_test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          path: './turbolift'
+      - name: install rustup and rust nightly
+        run: |
+          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly-2020-09-28
       - uses: engineerd/setup-kind@v0.5.0
         with:
           version: "v0.11.0"
+      - name: run tests
+        run: |
+          cd turbolift/examples/kubernetes_example
+          RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo test --features distributed -- --nocapture
+
+  kubernetes_example_distributed_run:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          path: './turbolift'
       - name: install rustup and rust nightly
         run: |
-            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly-2020-09-28
+          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly-2020-09-28
+      - uses: engineerd/setup-kind@v0.5.0
+        with:
+          version: "v0.11.0"
       - name: run tests
         run: |
-            cd turbolift/examples/kubernetes_example
-            sh run_tests.sh
+          cd turbolift/examples/kubernetes_example
+          RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run --features distributed
diff --git a/examples/kubernetes_example/run_distributed_tests.sh b/examples/kubernetes_example/run_distributed_tests.sh
deleted file mode 100644
index 5e9e3f12..00000000
--- a/examples/kubernetes_example/run_distributed_tests.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/usr/bin/env sh
-
-# error if any command fails
-set -e
-
-# run distributed tests
-RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo test --features distributed -- --nocapture
-RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run --features distributed
diff --git a/examples/kubernetes_example/run_non_distributed_tests.sh b/examples/kubernetes_example/run_non_distributed_tests.sh
deleted file mode 100644
index 3b4d7c2f..00000000
--- a/examples/kubernetes_example/run_non_distributed_tests.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/usr/bin/env sh
-
-# error if any command fails
-set -e
-
-# run non-distributed tests
-RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo test -- --nocapture
-RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run
\ No newline at end of file
diff --git a/examples/kubernetes_example/run_tests.sh b/examples/kubernetes_example/run_tests.sh
index 000d544d..202ee34d 100644
--- a/examples/kubernetes_example/run_tests.sh
+++ b/examples/kubernetes_example/run_tests.sh
@@ -11,13 +11,10 @@ printf "\n😤 deleting current cluster if it exists\n"
 kind delete cluster
 
 printf "\n📍 running non-distributed tests\n"
-. ./run_non_distributed_tests.sh
+RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo test -- --nocapture
+RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run
 echo "non-distributed tests completed."
 
-printf "\n🚽 deleting target folder to save space in CI\n"
-rm -r ./target
-echo "target folder deleted"
-
 printf "\n👷 setting up cluster with custom ingress-compatible config\n"
 cat <<EOF | kind create cluster --config=-
 kind: Cluster
@@ -53,15 +50,13 @@ kubectl wait --namespace ingress-nginx \
 echo "🚪 ingress ready."
 
 printf "\n🤸‍ run distributed tests\n"
-. ./run_distributed_tests.sh
+RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo test --features distributed -- --nocapture
+RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run --features distributed
 echo "🤸 distributed tests completed."
 
-printf "\n🚽 deleting target folder to save space in CI\n"
-rm -r ./target
-echo "target folder deleted"
-
 printf "\n📍 re-run non-distributed tests\n"
-. ./run_non_distributed_tests.sh
+RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo test -- --nocapture
+RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run
 echo "📍 non-distributed tests completed."
 
 printf "\n🚡turbolift tests complete.\n"

From dade1e3cfc1fd3bef8e54527a405b8df48d7d042 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sat, 12 Jun 2021 11:40:04 +0200
Subject: [PATCH 124/145] setup ingress in ci

---
 .github/workflows/examples.yml               |  2 ++
 examples/kubernetes_example/run_tests.sh     |  8 +-------
 examples/kubernetes_example/setup_ingress.sh | 10 ++++++++++
 3 files changed, 13 insertions(+), 7 deletions(-)
 create mode 100644 examples/kubernetes_example/setup_ingress.sh

diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml
index 11da9bb7..974a8a2c 100644
--- a/.github/workflows/examples.yml
+++ b/.github/workflows/examples.yml
@@ -57,6 +57,7 @@ jobs:
       - name: run tests
         run: |
           cd turbolift/examples/kubernetes_example
+          sh setup_ingress.sh
           RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo test --features distributed -- --nocapture
 
   kubernetes_example_distributed_run:
@@ -74,4 +75,5 @@ jobs:
       - name: run tests
         run: |
           cd turbolift/examples/kubernetes_example
+          sh setup_ingress.sh
           RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run --features distributed
diff --git a/examples/kubernetes_example/run_tests.sh b/examples/kubernetes_example/run_tests.sh
index 202ee34d..660a6bcf 100644
--- a/examples/kubernetes_example/run_tests.sh
+++ b/examples/kubernetes_example/run_tests.sh
@@ -39,13 +39,7 @@ kubectl cluster-info --context kind-kind
 echo "cluster initialized."
 
 printf "\n🚪 adding ingress\n"
-kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/static/provider/kind/deploy.yaml
-sleep 90s # give k8s time to generate the pod ^
-printf "\n⏱️ waiting for ingress controller to be ready...\n"
-kubectl wait --namespace ingress-nginx \
-  --for=condition=ready pod \
-  --selector=app.kubernetes.io/component=controller \
-  --timeout=30m
+. setup_ingress.sh
 
 echo "🚪 ingress ready."
 
diff --git a/examples/kubernetes_example/setup_ingress.sh b/examples/kubernetes_example/setup_ingress.sh
new file mode 100644
index 00000000..e83548d0
--- /dev/null
+++ b/examples/kubernetes_example/setup_ingress.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env sh
+set -e
+
+kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/static/provider/kind/deploy.yaml
+sleep 90s # give k8s time to generate the pod ^
+printf "\n⏱️ waiting for ingress controller to be ready...\n"
+kubectl wait --namespace ingress-nginx \
+  --for=condition=ready pod \
+  --selector=app.kubernetes.io/component=controller \
+  --timeout=30m

From edadbc7c86666b25cb49bef6439fbce3d6b2a97e Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sat, 12 Jun 2021 12:30:47 +0200
Subject: [PATCH 125/145] try increasing timeout for ingress

---
 examples/kubernetes_example/setup_ingress.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/kubernetes_example/setup_ingress.sh b/examples/kubernetes_example/setup_ingress.sh
index e83548d0..5c7b90bc 100644
--- a/examples/kubernetes_example/setup_ingress.sh
+++ b/examples/kubernetes_example/setup_ingress.sh
@@ -7,4 +7,4 @@ printf "\n⏱️ waiting for ingress controller to be ready...\n"
 kubectl wait --namespace ingress-nginx \
   --for=condition=ready pod \
   --selector=app.kubernetes.io/component=controller \
-  --timeout=30m
+  --timeout=50m

From a9608c3a8f66633982d4786c9c3b568176e14cf0 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sat, 12 Jun 2021 19:52:03 +0200
Subject: [PATCH 126/145] Revert "try increasing timeout for ingress"

This reverts commit edadbc7c86666b25cb49bef6439fbce3d6b2a97e.
---
 examples/kubernetes_example/setup_ingress.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/kubernetes_example/setup_ingress.sh b/examples/kubernetes_example/setup_ingress.sh
index 5c7b90bc..e83548d0 100644
--- a/examples/kubernetes_example/setup_ingress.sh
+++ b/examples/kubernetes_example/setup_ingress.sh
@@ -7,4 +7,4 @@ printf "\n⏱️ waiting for ingress controller to be ready...\n"
 kubectl wait --namespace ingress-nginx \
   --for=condition=ready pod \
   --selector=app.kubernetes.io/component=controller \
-  --timeout=50m
+  --timeout=30m

From b2ee4b0eeda397c389c622fcb852d9f625bf96cd Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sat, 12 Jun 2021 19:55:12 +0200
Subject: [PATCH 127/145] add debug statements to debug ci ingress issue

---
 examples/kubernetes_example/setup_ingress.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/kubernetes_example/setup_ingress.sh b/examples/kubernetes_example/setup_ingress.sh
index e83548d0..af396698 100644
--- a/examples/kubernetes_example/setup_ingress.sh
+++ b/examples/kubernetes_example/setup_ingress.sh
@@ -1,6 +1,4 @@
 #!/usr/bin/env sh
-set -e
-
 kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/static/provider/kind/deploy.yaml
 sleep 90s # give k8s time to generate the pod ^
 printf "\n⏱️ waiting for ingress controller to be ready...\n"
@@ -8,3 +6,5 @@ kubectl wait --namespace ingress-nginx \
   --for=condition=ready pod \
   --selector=app.kubernetes.io/component=controller \
   --timeout=30m
+kubectl describe pods -A
+kubectl logs --all-containers=true -l app.kubernetes.io/component=controller

From a0adf3f3cee086c726b386fa813c43a4f9eed993 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sat, 12 Jun 2021 20:13:25 +0200
Subject: [PATCH 128/145] add cluster instantiation and rename file from
 setup_ingress to setup_cluster

---
 .github/workflows/examples.yml               |  4 +-
 examples/kubernetes_example/run_tests.sh     | 30 +-------------
 examples/kubernetes_example/setup_cluster.sh | 43 ++++++++++++++++++++
 examples/kubernetes_example/setup_ingress.sh | 10 -----
 4 files changed, 47 insertions(+), 40 deletions(-)
 create mode 100644 examples/kubernetes_example/setup_cluster.sh
 delete mode 100644 examples/kubernetes_example/setup_ingress.sh

diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml
index 974a8a2c..0dffe94d 100644
--- a/.github/workflows/examples.yml
+++ b/.github/workflows/examples.yml
@@ -57,7 +57,7 @@ jobs:
       - name: run tests
         run: |
           cd turbolift/examples/kubernetes_example
-          sh setup_ingress.sh
+          sh setup_cluster.sh
           RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo test --features distributed -- --nocapture
 
   kubernetes_example_distributed_run:
@@ -75,5 +75,5 @@ jobs:
       - name: run tests
         run: |
           cd turbolift/examples/kubernetes_example
-          sh setup_ingress.sh
+          sh setup_cluster.sh
           RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run --features distributed
diff --git a/examples/kubernetes_example/run_tests.sh b/examples/kubernetes_example/run_tests.sh
index 660a6bcf..bdf1b56e 100644
--- a/examples/kubernetes_example/run_tests.sh
+++ b/examples/kubernetes_example/run_tests.sh
@@ -8,40 +8,14 @@ set -e
 echo "🚡 running turbolift tests..."
 
 printf "\n😤 deleting current cluster if it exists\n"
-kind delete cluster
+kind delete cluster # make sure we don't need the cluster when running locally
 
 printf "\n📍 running non-distributed tests\n"
 RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo test -- --nocapture
 RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run
 echo "non-distributed tests completed."
 
-printf "\n👷 setting up cluster with custom ingress-compatible config\n"
-cat <<EOF | kind create cluster --config=-
-kind: Cluster
-apiVersion: kind.x-k8s.io/v1alpha4
-nodes:
-- role: control-plane
-  kubeadmConfigPatches:
-  - |
-    kind: InitConfiguration
-    nodeRegistration:
-      kubeletExtraArgs:
-        node-labels: "ingress-ready=true"
-  extraPortMappings:
-  - containerPort: 80
-    hostPort: 80
-    protocol: TCP
-  - containerPort: 443
-    hostPort: 443
-    protocol: TCP
-EOF
-kubectl cluster-info --context kind-kind
-echo "cluster initialized."
-
-printf "\n🚪 adding ingress\n"
-. setup_ingress.sh
-
-echo "🚪 ingress ready."
+. setup_cluster.sh
 
 printf "\n🤸‍ run distributed tests\n"
 RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo test --features distributed -- --nocapture
diff --git a/examples/kubernetes_example/setup_cluster.sh b/examples/kubernetes_example/setup_cluster.sh
new file mode 100644
index 00000000..23b22b1b
--- /dev/null
+++ b/examples/kubernetes_example/setup_cluster.sh
@@ -0,0 +1,43 @@
+#!/usr/bin/env sh
+
+# assumes that kind is already setup
+
+printf "\n😤 deleting current cluster if it exists\n"
+kind delete cluster
+
+printf "\n👷 setting up cluster with custom ingress-compatible config\n"
+cat <<EOF | kind create cluster --config=-
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+  kubeadmConfigPatches:
+  - |
+    kind: InitConfiguration
+    nodeRegistration:
+      kubeletExtraArgs:
+        node-labels: "ingress-ready=true"
+  extraPortMappings:
+  - containerPort: 80
+    hostPort: 80
+    protocol: TCP
+  - containerPort: 443
+    hostPort: 443
+    protocol: TCP
+EOF
+kubectl cluster-info --context kind-kind
+echo "cluster initialized."
+
+
+printf "\n🚪 adding ingress\n"
+kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/static/provider/kind/deploy.yaml
+sleep 90s # give k8s time to generate the pod ^
+printf "\n⏱️ waiting for ingress controller to be ready...\n"
+kubectl wait --namespace ingress-nginx \
+  --for=condition=ready pod \
+  --selector=app.kubernetes.io/component=controller \
+  --timeout=30m
+echo "🚪 ingress ready."
+
+kubectl describe pods -A
+kubectl logs --all-containers=true -l app.kubernetes.io/component=controller
diff --git a/examples/kubernetes_example/setup_ingress.sh b/examples/kubernetes_example/setup_ingress.sh
deleted file mode 100644
index af396698..00000000
--- a/examples/kubernetes_example/setup_ingress.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/usr/bin/env sh
-kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/static/provider/kind/deploy.yaml
-sleep 90s # give k8s time to generate the pod ^
-printf "\n⏱️ waiting for ingress controller to be ready...\n"
-kubectl wait --namespace ingress-nginx \
-  --for=condition=ready pod \
-  --selector=app.kubernetes.io/component=controller \
-  --timeout=30m
-kubectl describe pods -A
-kubectl logs --all-containers=true -l app.kubernetes.io/component=controller

From c11b79cdc90d4c317b7e6da547e2cf74d2228d38 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sat, 12 Jun 2021 22:31:05 +0200
Subject: [PATCH 129/145] Revert "add debug statements to debug ci ingress
 issue"

This reverts commit b2ee4b0eeda397c389c622fcb852d9f625bf96cd.
---
 examples/kubernetes_example/setup_cluster.sh | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/examples/kubernetes_example/setup_cluster.sh b/examples/kubernetes_example/setup_cluster.sh
index 23b22b1b..e9c2e7bb 100644
--- a/examples/kubernetes_example/setup_cluster.sh
+++ b/examples/kubernetes_example/setup_cluster.sh
@@ -38,6 +38,3 @@ kubectl wait --namespace ingress-nginx \
   --selector=app.kubernetes.io/component=controller \
   --timeout=30m
 echo "🚪 ingress ready."
-
-kubectl describe pods -A
-kubectl logs --all-containers=true -l app.kubernetes.io/component=controller

From 7626051397aacc116fa1b863c53391c2ddd11c3c Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sat, 12 Jun 2021 23:09:18 +0200
Subject: [PATCH 130/145] remove debug prints

---
 turbolift_internals/src/kubernetes.rs  | 27 --------------------------
 turbolift_internals/src/local_queue.rs |  7 ++-----
 turbolift_macros/src/lib.rs            | 19 ------------------
 3 files changed, 2 insertions(+), 51 deletions(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 96fb1242..eafc821b 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -114,10 +114,6 @@ impl DistributionPlatform for K8s {
         let ingress_name = format!("{}-ingress", app_name);
         let tag_in_reg = make_image(self, &app_name, function_name, project_tar)?;
 
-        println!("image made. making deployment and service names.");
-        println!("made service_name");
-        println!("made app_name and container_name");
-
         // make deployment
         let deployment_json = serde_json::json!({
             "apiVersion": "apps/v1",
@@ -150,14 +146,11 @@ impl DistributionPlatform for K8s {
                 }
             }
         });
-        println!("deployment_json generated");
         let deployment = serde_json::from_value(deployment_json)?;
-        println!("deployment generated");
         deployments
             .create(&PostParams::default(), &deployment)
             .compat()
             .await?;
-        println!("created deployment");
 
         // make service pointing to deployment
         let service_json = serde_json::json!({
@@ -177,12 +170,10 @@ impl DistributionPlatform for K8s {
             }
         });
         let service = serde_json::from_value(service_json)?;
-        println!("deployment generated");
         services
             .create(&PostParams::default(), &service)
             .compat()
             .await?;
-        println!("created service");
 
         // make ingress pointing to service
         let ingress = serde_json::json!({
@@ -231,17 +222,6 @@ impl DistributionPlatform for K8s {
             )
         }
 
-        let node_ip = {
-            // let stdout = Command::new("kubectl")
-            // .args("get nodes --selector=kubernetes.io/role!=master -o jsonpath={.items[*].status.addresses[?\\(@.type==\\\"InternalIP\\\"\\)].address}".split(' '))
-            // .output()
-            // .expect("error finding node ip")
-            // .stdout;
-            // String::from_utf8(stdout).expect("could not parse local node ip")
-            "localhost".to_string()
-        };
-        println!("found node ip: {}", node_ip.as_str());
-
         // let node_port: i32 = 5000;
         // let node_port = service
         //     .spec
@@ -257,7 +237,6 @@ impl DistributionPlatform for K8s {
             "http://localhost:{}/{}/{}/",
             EXTERNAL_PORT, function_name, run_id
         );
-        println!("generated service_ip: {}", service_ip.as_str());
 
         // todo make sure that the pod and service were correctly started before returning
 
@@ -286,8 +265,6 @@ impl DistributionPlatform for K8s {
 
         self.fn_names_to_services
             .insert(function_name.to_string(), Url::from_str(&service_ip)?);
-
-        println!("returning from declare");
         Ok(())
     }
 
@@ -302,7 +279,6 @@ impl DistributionPlatform for K8s {
         let args = format!("./{}", params);
         let query_url = service_base_url.join(&args)?;
         tracing::info!(url = query_url.as_str(), "sending dispatch request");
-        println!("sending dispatch request to {}", query_url.as_str());
         Ok(self
             .request_client
             .get(query_url)
@@ -316,7 +292,6 @@ impl DistributionPlatform for K8s {
 
     #[tracing::instrument]
     fn has_declared(&self, fn_name: &str) -> bool {
-        println!("in has_declared");
         self.fn_names_to_services.contains_key(fn_name)
     }
 }
@@ -417,10 +392,8 @@ CMD [\"./{function_name}\", \"0.0.0.0:{container_port}\"]",
 
         Ok(unique_tag.clone())
     })();
-    println!("removing build dir");
     // always remove the build directory, even on build error
     std::fs::remove_dir_all(build_dir_canonical)?;
-    println!("returning result");
 
     result.and((k8s.deploy_container)(unique_tag.as_str()).map(|s| s.to_string()))
 }
diff --git a/turbolift_internals/src/local_queue.rs b/turbolift_internals/src/local_queue.rs
index 98721d0f..d4ff2001 100644
--- a/turbolift_internals/src/local_queue.rs
+++ b/turbolift_internals/src/local_queue.rs
@@ -103,8 +103,7 @@ impl DistributionPlatform for LocalQueue {
         let query_url = address_and_port.join(&prefixed_params)?;
 
         tracing::info!("sending dispatch request");
-        println!("sending dispatch request to {}", query_url.as_str());
-        let resp = Ok(self
+        Ok(self
             .request_client
             .get(query_url)
             .send()
@@ -112,9 +111,7 @@ impl DistributionPlatform for LocalQueue {
             .await?
             .text()
             .compat()
-            .await?);
-        println!("received response");
-        resp
+            .await?)
     }
 
     #[tracing::instrument]
diff --git a/turbolift_macros/src/lib.rs b/turbolift_macros/src/lib.rs
index 381698c4..f8a29f6a 100644
--- a/turbolift_macros/src/lib.rs
+++ b/turbolift_macros/src/lib.rs
@@ -184,14 +184,6 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
         let tar = extract_function::make_compressed_proj_src(&function_cache_proj_path);
         let tar_file = CACHE_PATH.join(original_target_function_name.clone() + "_source.tar");
         fs::write(&tar_file, tar).expect("failure writing bin");
-        println!(
-            "tar file location: {}",
-            tar_file
-                .canonicalize()
-                .expect("error canonicalizing tar file location")
-                .to_str()
-                .unwrap()
-        );
         TokenStream2::from_str(&format!(
             "std::include_bytes!(\"{}\")",
             tar_file
@@ -202,7 +194,6 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
         ))
         .expect("syntax error while embedding project tar.")
     };
-    println!("project_source_binary complete");
 
     // generate API function for the microservice
     let declare_and_dispatch = q! {
@@ -218,24 +209,16 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
             use turbolift::tokio_compat_02::FutureExt;
             use turbolift::uuid::Uuid;
 
-            println!("in original target function");
-
             let mut platform = #distribution_platform.lock().await;
 
-            println!("platform acquired");
-
             if !platform.has_declared(#original_target_function_name) {
-                println!("launching declare");
                 platform
                     .declare(#original_target_function_name, Uuid::new_v4(), #project_source_binary)
                     .compat()
                     .await?;
-                println!("declare completed");
             }
 
             let params = #params_vec.join("/");
-
-            println!("launching dispatch");
             let resp_string = platform
                 .dispatch(
                     #original_target_function_name,
@@ -243,8 +226,6 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
                 )
                 .compat()
                 .await?;
-            println!("dispatch completed");
-            println!("resp_string: {}", &resp_string);
             Ok(turbolift::serde_json::from_str(&resp_string)?)
         }
     };

From f665ce4b746999fd30ff0366abd104a4b54798cb Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sat, 12 Jun 2021 23:11:03 +0200
Subject: [PATCH 131/145] re-enable autoscale code

---
 turbolift_internals/src/kubernetes.rs | 38 +++++++++++++--------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index eafc821b..c09b894e 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -240,25 +240,25 @@ impl DistributionPlatform for K8s {
 
         // todo make sure that the pod and service were correctly started before returning
 
-        // if self.max_scale_n > 1 {
-        //     // set autoscale
-        //     let scale_args = format!(
-        //         "autoscale deployment {} --max={}",
-        //         deployment_name, self.max_scale_n
-        //     );
-        //     let scale_status = Command::new("kubectl")
-        //         .args(scale_args.as_str().split(' '))
-        //         .status()?;
-        //
-        //     if !scale_status.success() {
-        //         return Err(anyhow::anyhow!(
-        //             "autoscale error: error code: {:?}",
-        //             scale_status.code()
-        //         )
-        //         .into());
-        //         // ^ todo attach error context from child
-        //     }
-        // }
+        if self.max_scale_n > 1 {
+            // set autoscale
+            let scale_args = format!(
+                "autoscale deployment {} --max={}",
+                deployment_name, self.max_scale_n
+            );
+            let scale_status = Command::new("kubectl")
+                .args(scale_args.as_str().split(' '))
+                .status()?;
+
+            if !scale_status.success() {
+                return Err(anyhow::anyhow!(
+                    "autoscale error: error code: {:?}",
+                    scale_status.code()
+                )
+                .into());
+                // ^ todo attach error context from child
+            }
+        }
 
         sleep(Duration::from_secs(90)).await;
         // todo implement the check on whether the service is running / pod failed

From b590ab7b25fabf514b774eaaadf660a6d17e086b Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sat, 12 Jun 2021 23:12:26 +0200
Subject: [PATCH 132/145] consolidate related todo comments

---
 turbolift_internals/src/kubernetes.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index c09b894e..1d4237e6 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -238,8 +238,6 @@ impl DistributionPlatform for K8s {
             EXTERNAL_PORT, function_name, run_id
         );
 
-        // todo make sure that the pod and service were correctly started before returning
-
         if self.max_scale_n > 1 {
             // set autoscale
             let scale_args = format!(
@@ -261,6 +259,7 @@ impl DistributionPlatform for K8s {
         }
 
         sleep(Duration::from_secs(90)).await;
+        // todo make sure that the pod and service were correctly started before returning
         // todo implement the check on whether the service is running / pod failed
 
         self.fn_names_to_services

From 6fcd8ee25cd649d05ab533e4869bf9e6ff494898 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sat, 12 Jun 2021 23:14:19 +0200
Subject: [PATCH 133/145] don't call ingress ips service ips

---
 turbolift_internals/src/kubernetes.rs | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 1d4237e6..b5e6a492 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -45,7 +45,7 @@ pub const TARGET_ARCHITECTURE: Option<&str> = None;
 /// [try_default()](https://docs.rs/kube/0.56.0/kube/client/struct.Client.html#method.try_default).
 pub struct K8s {
     max_scale_n: u32,
-    fn_names_to_services: HashMap<String, Url>,
+    fn_names_to_ips: HashMap<String, Url>,
     request_client: reqwest::Client,
 
     #[derivative(Debug = "ignore")]
@@ -80,7 +80,7 @@ impl K8s {
         K8s {
             deploy_container,
             max_scale_n: max,
-            fn_names_to_services: HashMap::new(),
+            fn_names_to_ips: HashMap::new(),
             request_client: reqwest::Client::new(),
         }
     }
@@ -233,7 +233,7 @@ impl DistributionPlatform for K8s {
         //     .next()
         //     .expect("no node port assigned to service");
         // let service_ip = format!("http://{}", node_ip, node_port);
-        let service_ip = format!(
+        let ingress_ip = format!(
             "http://localhost:{}/{}/{}/",
             EXTERNAL_PORT, function_name, run_id
         );
@@ -262,8 +262,8 @@ impl DistributionPlatform for K8s {
         // todo make sure that the pod and service were correctly started before returning
         // todo implement the check on whether the service is running / pod failed
 
-        self.fn_names_to_services
-            .insert(function_name.to_string(), Url::from_str(&service_ip)?);
+        self.fn_names_to_ips
+            .insert(function_name.to_string(), Url::from_str(&ingress_ip)?);
         Ok(())
     }
 
@@ -274,7 +274,7 @@ impl DistributionPlatform for K8s {
         params: ArgsString,
     ) -> DistributionResult<JsonResponse> {
         // request from server
-        let service_base_url = self.fn_names_to_services.get(function_name).unwrap();
+        let service_base_url = self.fn_names_to_ips.get(function_name).unwrap();
         let args = format!("./{}", params);
         let query_url = service_base_url.join(&args)?;
         tracing::info!(url = query_url.as_str(), "sending dispatch request");
@@ -291,7 +291,7 @@ impl DistributionPlatform for K8s {
 
     #[tracing::instrument]
     fn has_declared(&self, fn_name: &str) -> bool {
-        self.fn_names_to_services.contains_key(fn_name)
+        self.fn_names_to_ips.contains_key(fn_name)
     }
 }
 

From f2b2aae249a37d751e3704f09af4d9f1d6ec8d50 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sat, 12 Jun 2021 23:16:48 +0200
Subject: [PATCH 134/145] remove irrelevant, commented out code

---
 turbolift_internals/src/kubernetes.rs | 13 +------------
 1 file changed, 1 insertion(+), 12 deletions(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index b5e6a492..fe93373f 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -222,21 +222,10 @@ impl DistributionPlatform for K8s {
             )
         }
 
-        // let node_port: i32 = 5000;
-        // let node_port = service
-        //     .spec
-        //     .expect("no specification found for service")
-        //     .ports
-        //     .expect("no ports found for service")
-        //     .iter()
-        //     .filter_map(|port| port.node_port)
-        //     .next()
-        //     .expect("no node port assigned to service");
-        // let service_ip = format!("http://{}", node_ip, node_port);
         let ingress_ip = format!(
             "http://localhost:{}/{}/{}/",
             EXTERNAL_PORT, function_name, run_id
-        );
+        ); // we assume for now that the ingress is exposed on localhost
 
         if self.max_scale_n > 1 {
             // set autoscale

From 7f6d6a8b8c3e435de5c03f8fc26603667d391b4b Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sun, 13 Jun 2021 09:44:51 +0200
Subject: [PATCH 135/145] fix nightly version in local queue test

---
 examples/local_queue_example/Dockerfile | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/examples/local_queue_example/Dockerfile b/examples/local_queue_example/Dockerfile
index b8d94e70..554a011c 100644
--- a/examples/local_queue_example/Dockerfile
+++ b/examples/local_queue_example/Dockerfile
@@ -1,12 +1,14 @@
 # run me in turbolift root! E.G.: "docker build -f examples/local_queue_example/Dockerfile ."
 FROM rustlang/rust:nightly
+RUN rustup default nightly-2020-09-28
+ENV RUSTFLAGS='--cfg procmacro2_semver_exempt'
 COPY ./ turbolift
 WORKDIR turbolift/examples/local_queue_example
 
 # test
-RUN RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test -- --nocapture
-RUN RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly test --features distributed -- --nocapture
+RUN cargo +nightly test -- --nocapture
+RUN cargo +nightly test --features distributed -- --nocapture
 
 # run
-RUN RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run
-RUN RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo +nightly run --features distributed
+RUN cargo +nightly run
+RUN cargo +nightly run --features distributed

From cb196886a0067171113483afd2beb8a19e0562a1 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sun, 13 Jun 2021 10:08:08 +0200
Subject: [PATCH 136/145] remove double import

---
 turbolift_internals/src/extract_function.rs | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/turbolift_internals/src/extract_function.rs b/turbolift_internals/src/extract_function.rs
index e9cc15e2..0808dcf1 100644
--- a/turbolift_internals/src/extract_function.rs
+++ b/turbolift_internals/src/extract_function.rs
@@ -1,4 +1,3 @@
-use proc_macro2::TokenStream;
 use std::collections::VecDeque;
 use std::fs;
 use std::io::Cursor;
@@ -19,7 +18,7 @@ type ParamTypes = syn::punctuated::Punctuated<Box<syn::Type>, syn::Token![,]>;
 const IGNORED_DIRECTORIES: [&str; 3] = ["target", ".git", ".turbolift"];
 
 #[tracing::instrument]
-pub fn get_fn_item(function: TokenStream) -> syn::ItemFn {
+pub fn get_fn_item(function: TokenStream2) -> syn::ItemFn {
     match syn::parse2(function).unwrap() {
         syn::Item::Fn(fn_item) => fn_item,
         _ => panic!("token stream does not represent function."),
@@ -109,7 +108,7 @@ pub fn to_param_types(typed_params: TypedParams) -> ParamTypes {
 }
 
 #[tracing::instrument]
-pub fn params_json_vec(untyped_params: UntypedParams) -> TokenStream {
+pub fn params_json_vec(untyped_params: UntypedParams) -> TokenStream2 {
     let punc: Vec<String> = untyped_params
         .into_iter()
         .map(|pat| {
@@ -120,11 +119,11 @@ pub fn params_json_vec(untyped_params: UntypedParams) -> TokenStream {
         .collect();
 
     let vec_string = format!("vec![{}]", punc.join(", "));
-    TokenStream::from_str(&vec_string).unwrap()
+    TokenStream2::from_str(&vec_string).unwrap()
 }
 
 #[tracing::instrument]
-pub fn get_sanitized_file(function: &TokenStream) -> TokenStream {
+pub fn get_sanitized_file(function: &TokenStream2) -> TokenStream2 {
     let span = function.span();
     let path = span.source_file().path();
     let start_line = match span.start().line {
@@ -159,10 +158,10 @@ pub fn get_sanitized_file(function: &TokenStream) -> TokenStream {
 }
 
 #[tracing::instrument]
-pub fn unpack_path_params(untyped_params: &UntypedParams) -> TokenStream {
+pub fn unpack_path_params(untyped_params: &UntypedParams) -> TokenStream2 {
     let n_params = untyped_params.len();
     let params: Vec<String> = (0..n_params).map(|i| format!("path.{}", i)).collect();
-    TokenStream::from_str(&params.join(", ")).unwrap()
+    TokenStream2::from_str(&params.join(", ")).unwrap()
 }
 
 #[tracing::instrument]

From 55c350220cfeb902897fae6dcb0543e99843ea66 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sun, 13 Jun 2021 11:15:43 +0200
Subject: [PATCH 137/145] reorder imports for clarity

---
 turbolift_internals/src/kubernetes.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index fe93373f..1c6aad39 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -1,4 +1,5 @@
 use std::collections::HashMap;
+use std::io::Write;
 use std::process::{Command, Stdio};
 use std::str::FromStr;
 
@@ -12,14 +13,13 @@ use regex::Regex;
 use tokio::time::{sleep, Duration};
 use tokio_compat_02::FutureExt;
 use url::Url;
+use uuid::Uuid;
 
 use crate::distributed_platform::{
     ArgsString, DistributionPlatform, DistributionResult, JsonResponse,
 };
 use crate::utils::{DEBUG_FLAG, RELEASE_FLAG};
 use crate::CACHE_PATH;
-use std::io::Write;
-use uuid::Uuid;
 
 const TURBOLIFT_K8S_NAMESPACE: &str = "default";
 type ImageTag = String;

From e1f19466aeca5616d1d62cefd566de5c418a4a5a Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sun, 13 Jun 2021 14:25:22 +0200
Subject: [PATCH 138/145] upgrade to version 0.1.1

---
 Cargo.toml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 3e17199f..0003934c 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,10 +1,11 @@
 [package]
 name = "turbolift"
-version = "0.1.0"
+version = "0.1.1"
 authors = ["Dominic Burkart <@DominicBurkart>"]
 edition = "2018"
-description = "WIP"
+description = "Easy distribution interface"
 readme = "README.md"
+homepage = "https://dominic.computer/turbolift"
 license = "Hippocratic-2.1"
 
 [features]

From 987eaf098b9d12d00c49d2e29092316623a85684 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sun, 13 Jun 2021 14:54:10 +0200
Subject: [PATCH 139/145] update readme

---
 README.md | 114 +++++++++++++++++++++++++++---------------------------
 1 file changed, 58 insertions(+), 56 deletions(-)

diff --git a/README.md b/README.md
index cac0f7ff..5cf6856f 100644
--- a/README.md
+++ b/README.md
@@ -4,82 +4,84 @@
     src="https://img.shields.io/crates/v/turbolift.svg"
     alt="turbolift’s current version badge"
     title="turbolift’s current version badge" />
-[![status](https://github.com/DominicBurkart/turbolift/workflows/rust/badge.svg)](https://github.com/DominicBurkart/turbolift/actions?query=is%3Acompleted+branch%3Amaster+workflow%3A"rust")
-[![status](https://github.com/DominicBurkart/turbolift/workflows/docker/badge.svg)](https://github.com/DominicBurkart/turbolift/actions?query=is%3Acompleted+branch%3Amaster+workflow%3A"docker")
+[![status](https://img.shields.io/github/checks-status/dominicburkart/turbolift/master)](https://github.com/DominicBurkart/turbolift/actions?query=branch%3Amaster)
 
-Turbolift is a WIP distribution platform for rust. It's designed to make distribution an afterthought 
-by extracting and distributing specific functions and their dependencies from a larger rust application.
-Turbolift then acts as the glue between these extracted mini-apps and the main application.
+Turbolift is a WIP distribution platform for rust. It's designed to make
+distribution an afterthought by extracting and distributing specific
+functions and their dependencies from a larger rust application.
+Turbolift then acts as the glue between these extracted mini-apps and
+the main application.
 
-Look in the [examples](https://github.com/DominicBurkart/turbolift/tree/master/examples) directory for 
-full projects with working syntax examples. 
+Look in the [examples](https://github.com/DominicBurkart/turbolift/tree/master/examples)
+directory for full projects with working syntax examples.
 
-## Distribution as an afterthought.
-Turbolift allows developers to turn normal rust functions into distributed services 
- just by tagging them with a macro. This lets you develop in a monorepo environment, 
-but benefit from the scalability of microservice architectures. Right now, Turbolift 
-only works with K8s, though it's designed to be extended to other cluster management utilities.
+## Distribution as an afterthought
 
-## Orchestration with a feature flag.
-For quicker development builds, `cargo build` doesn't build the distributed version of your code by default.
-Distribution is feature-gated so that it's easy to turn on (for example, in production), or off (for example, 
-while developing locally). Enabling a dependency feature is as simple as adding a couple of lines to your `Cargo.toml`. 
-Check out the examples directory to see how to add a new feature that triggers turbolift to run in distributed mode
-when activated.
+Turbolift allows developers to turn normal rust functions into distributed services
+just by tagging them with a macro. Right now, Turbolift only works with K8s, though
+it's designed to be extended to other cluster management utilities.
+
+## Orchestration with a feature flag
+
+Distribution is feature-gated in Turbolift, so it's easy to activate distribution
+for some builds and deactivate it for others (while developing locally, for
+example).
 
 ## Important implementation notes
-- implemented over http using `reqwest` and `actix-web` (no current plans to refactor to use a lower level network protocol).
-- assumes a secure network– function parameters are sent in plaintext to the microservice.
-- source vulnerability: when building, anything in the project directory or in local dependencies 
-declared in the project manifest could be bundled and sent over the network to workers. 
+
+- implemented over http using `reqwest` and `actix-web` (no current plans to
+refactor to use a lower level network protocol).
+- assumes a secure network– function parameters are sent in plaintext to the
+microservice.
+- source vulnerability: when building, anything in the project directory or in
+local dependencies declared in the project manifest could be bundled and sent
+over the network to workers.
+
+More information is available on the [project homepage](https://dominic.computer/turbolift).
 
 ## Current Limitations
-- *Because of reliance on unstable proc_macro::Span features, all programs using turbolift need to 
-be built with an unstable nightly compiler flag (e.g. `RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo build`)* ([tracking issue](https://github.com/rust-lang/rust/issues/54725)).
-- Functions are assumed to be pure (lacking side-effects such as 
-writing to the file system or mutation of a global variable). 
-*Today, this is not enforced by the code.* 
-- For a function to be distributed, its inputs and outputs have to be (de)serializable with [Serde](https://github.com/serde-rs/serde).
+
+- DO NOT RUN TURBOLIFT ON A PUBLIC-FACING CLUSTER.
+- *Because of reliance on unstable proc_macro::Span features, all programs
+using turbolift need to be built with an unstable nightly compiler flag (e.g.
+`RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo build`)*
+([tracking issue](https://github.com/rust-lang/rust/issues/54725)).
+- Functions are assumed to be pure (lacking side-effects such as
+writing to the file system or mutation of a global variable).
+*Today, this is not enforced by the code.*
+- For a function to be distributed, its inputs and outputs have to be
+(de)serializable with [Serde](https://github.com/serde-rs/serde).
 - Distributed functions cannot be nested in other functions.
 - Distributed functions cannot be methods.
 - Distributed functions cannot use other functions called `main`.
-- Distributed functions not in `main.rs` cannot use functions declared 
+- Distributed functions not in `main.rs` cannot use functions declared
 in `main.rs`.
 - Distributed functions cannot have `-> impl Trait` types.
-- Unused functions that have been marked with the `on` macro will still be 
-compiled for distribution, even if eventually the linker will then 
+- Unused functions that have been marked with the `on` macro will still be
+compiled for distribution, even if eventually the linker will then
 remove the completed binary and distribution code.
-- *Turbolift doesn't match the cargo compilation settings for microservices yet.*
-- projects can have relative local dependencies listing in the cargo manifest, but those dependencies themselves 
-should not have relative local dependencies prone to breaking.
-- if your program produces side effects when initialized, for example when 
-global constants are initialized, those side effects may be triggered 
+- projects can have relative local dependencies listing in the cargo
+manifest, but those dependencies themselves should not have relative local
+dependencies prone to breaking.
+- if your program produces side effects when initialized, for example when
+global constants are initialized, those side effects may be triggered
 for each function call.
-- currently, turbolift does not guarantee that the target operating system for a 
-program will also be used with its microservices.
-- currently, turbolift's k8s support only works with the en0 or eth0 interface. For devices 
-with multiple network interfaces (en1, en2, ..., eth1, eth2, ..., and other kinds of network 
-interfaces), we choose the first en0 or eth0 interface and ignore all other interfaces
-while sharing information from the across the local network. Instead, we should allow users to 
-choose the relevant network, or infer it based on what the k8s instance is using. 
-- When running k8s cluster, a local registry is set up on the host machine 
-by using the registry image. Distribution is handled by this registry, which currently 
-does not handle auto-deletion of distributed images. This means that the image must occasionally 
-be wiped if the same environment is reused to distribute many functions over time.
-- Currently, it is assumed that all nodes are on the same local network. This is important 
-for distributing images for each extracted function without using a private repository. We 
-need private repository support if we want to support k8s instances where all nodes 
-are not on a single, local network.
+- turbolift runs functions on an unreproducible linux build, it doesn't
+e.g. pin the env or match the OS of the current environment.
 
 ## Current Project Goals
-- [ ] support kubernetes ([pr](https://github.com/DominicBurkart/turbolift/pull/2)).
+
+- [X] support kubernetes ([pr](https://github.com/DominicBurkart/turbolift/pull/2)).
 - [ ] roadmap support for other targets.
-- [X] only use distributed configuration when flagged (like in `cargo build --features "distributed"`). Otherwise,
-just transform the tagged function into an async function (to provide an identical API), but don't 
-build any microservices or alter any code.
-- [ ] build cross-architecture compilation tests into the CI (RN we only test via github actions read Docker, and a different custom Docker test workflow)
+- [X] only use distributed configuration when flagged (like in
+`cargo build --features "distributed"`). Otherwise, just transform the
+tagged function into an async function (to provide an identical API), but
+don't build any microservices or alter any code.
+- [ ] build cross-architecture compilation tests into the CI.
 
 ## Current tech debt todo
+
 - [ ] start reducing ginormous API, right now basically everything is public
 - [ ] refactor split between turbolift_internals and turbolift_macros
 - [ ] improve names
+- [ ] send params in json as payload instead of directly in the url

From 34cd2da951244e06fd15131439276e1bd0af3471 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sun, 13 Jun 2021 15:15:55 +0200
Subject: [PATCH 140/145] cleanup all run resources when global state handler
 is dropped

note: does not delete local docker image, or docker image in cluster
---
 .../src/distributed_platform.rs               |  8 +-
 turbolift_internals/src/kubernetes.rs         | 82 +++++++------------
 turbolift_internals/src/local_queue.rs        | 10 +--
 turbolift_macros/src/lib.rs                   |  2 +-
 4 files changed, 36 insertions(+), 66 deletions(-)

diff --git a/turbolift_internals/src/distributed_platform.rs b/turbolift_internals/src/distributed_platform.rs
index 59944b88..ba0bfce7 100644
--- a/turbolift_internals/src/distributed_platform.rs
+++ b/turbolift_internals/src/distributed_platform.rs
@@ -1,7 +1,6 @@
 extern crate proc_macro;
 use async_trait::async_trait;
 use std::error;
-use uuid::Uuid;
 
 pub type DistributionError = Box<dyn error::Error>;
 pub type DistributionResult<T> = std::result::Result<T, DistributionError>;
@@ -12,12 +11,7 @@ pub type JsonResponse = String;
 #[async_trait]
 pub trait DistributionPlatform {
     /// declare a function
-    async fn declare(
-        &mut self,
-        function_name: &str,
-        run_id: Uuid,
-        project_tar: &[u8],
-    ) -> DistributionResult<()>;
+    async fn declare(&mut self, function_name: &str, project_tar: &[u8]) -> DistributionResult<()>;
 
     // dispatch params to a function
     async fn dispatch(
diff --git a/turbolift_internals/src/kubernetes.rs b/turbolift_internals/src/kubernetes.rs
index 1c6aad39..5f288a9e 100644
--- a/turbolift_internals/src/kubernetes.rs
+++ b/turbolift_internals/src/kubernetes.rs
@@ -47,6 +47,7 @@ pub struct K8s {
     max_scale_n: u32,
     fn_names_to_ips: HashMap<String, Url>,
     request_client: reqwest::Client,
+    run_id: Uuid,
 
     #[derivative(Debug = "ignore")]
     /// A function called after the image is built locally via docker. deploy_container
@@ -82,6 +83,7 @@ impl K8s {
             max_scale_n: max,
             fn_names_to_ips: HashMap::new(),
             request_client: reqwest::Client::new(),
+            run_id: Uuid::new_v4(),
         }
     }
 }
@@ -93,12 +95,7 @@ fn sanitize_function_name(function_name: &str) -> String {
 #[async_trait]
 impl DistributionPlatform for K8s {
     #[tracing::instrument(skip(project_tar))]
-    async fn declare(
-        &mut self,
-        function_name: &str,
-        run_id: Uuid,
-        project_tar: &[u8],
-    ) -> DistributionResult<()> {
+    async fn declare(&mut self, function_name: &str, project_tar: &[u8]) -> DistributionResult<()> {
         // connect to cluster. tries in-cluster configuration first, then falls back to kubeconfig file.
         let deployment_client = Client::try_default().compat().await?;
         let deployments: Api<Deployment> =
@@ -107,7 +104,7 @@ impl DistributionPlatform for K8s {
         let services: Api<Service> = Api::namespaced(service_client, TURBOLIFT_K8S_NAMESPACE);
 
         // generate image & push
-        let app_name = format!("{}-{}", sanitize_function_name(function_name), run_id);
+        let app_name = format!("{}-{}", sanitize_function_name(function_name), self.run_id);
         let container_name = format!("{}-app", app_name);
         let deployment_name = format!("{}-deployment", app_name);
         let service_name = format!("{}-service", app_name);
@@ -120,6 +117,9 @@ impl DistributionPlatform for K8s {
             "kind": "Deployment",
             "metadata": {
                 "name": deployment_name,
+                "labels": {
+                    "turbolift_run_id": self.run_id.to_string()
+                }
             },
             "spec": {
                 "selector": {
@@ -132,7 +132,8 @@ impl DistributionPlatform for K8s {
                     "metadata": {
                      "name": format!("{}-app", app_name),
                      "labels": {
-                       "app": app_name
+                       "app": app_name,
+                       "turbolift_run_id": self.run_id.to_string(),
                      }
                     },
                     "spec": {
@@ -158,6 +159,9 @@ impl DistributionPlatform for K8s {
             "kind": "Service",
             "metadata": {
                 "name": service_name,
+                "labels": {
+                    "turbolift_run_id": self.run_id.to_string(),
+                }
             },
             "spec": {
                 "selector": {
@@ -180,7 +184,10 @@ impl DistributionPlatform for K8s {
             "apiVersion": "networking.k8s.io/v1",
             "kind": "Ingress",
             "metadata": {
-                "name": ingress_name
+                "name": ingress_name,
+                "labels": {
+                    "turbolift_run_id": self.run_id.to_string(),
+                }
             },
             "spec": {
                 "rules": [
@@ -188,7 +195,7 @@ impl DistributionPlatform for K8s {
                         "http": {
                             "paths": [
                                 {
-                                    "path": format!("/{}/{}", function_name, run_id),
+                                    "path": format!("/{}/{}", function_name, self.run_id),
                                     "pathType": "Prefix",
                                     "backend": {
                                         "service" : {
@@ -224,7 +231,7 @@ impl DistributionPlatform for K8s {
 
         let ingress_ip = format!(
             "http://localhost:{}/{}/{}/",
-            EXTERNAL_PORT, function_name, run_id
+            EXTERNAL_PORT, function_name, self.run_id
         ); // we assume for now that the ingress is exposed on localhost
 
         if self.max_scale_n > 1 {
@@ -389,45 +396,18 @@ CMD [\"./{function_name}\", \"0.0.0.0:{container_port}\"]",
 impl Drop for K8s {
     #[tracing::instrument]
     fn drop(&mut self) {
-        // todo
-
-        // delete the associated services and deployments from the functions we distributed
-        // let rt = tokio::runtime::Runtime::new().unwrap();
-        // rt.block_on(async {
-        //     let deployment_client = Client::try_default().compat().await.unwrap();
-        //     let deployments: Api<Deployment> =
-        //         Api::namespaced(deployment_client, TURBOLIFT_K8S_NAMESPACE);
-        //     let service_client = Client::try_default().compat().await.unwrap();
-        //     let services: Api<Service> = Api::namespaced(service_client, TURBOLIFT_K8S_NAMESPACE);
-        //
-        //     let distributed_functions = self.fn_names_to_services.keys();
-        //     for function in distributed_functions {
-        //         let service = function_to_service_name(function);
-        //         services
-        //             .delete(&service, &Default::default())
-        //             .compat()
-        //             .await
-        //             .unwrap();
-        //         let deployment = function_to_deployment_name(function);
-        //         deployments
-        //             .delete(&deployment, &Default::default())
-        //             .compat()
-        //             .await
-        //             .unwrap();
-        //     }
-        // });
-        //
-        // // delete the local registry
-        // let registry_deletion_status = Command::new("docker")
-        //     .arg("rmi")
-        //     .arg("$(docker images |grep 'turbolift-registry')")
-        //     .status()
-        //     .unwrap();
-        // if !registry_deletion_status.success() {
-        //     eprintln!(
-        //         "could not delete turblift registry docker image. error code: {}",
-        //         registry_deletion_status.code().unwrap()
-        //     );
-        // }
+        let status = Command::new("kubectl")
+            .args(
+                format!(
+                    "delete pods,deployments,services,ingress -l turbolift_run_id={}",
+                    self.run_id.to_string()
+                )
+                .split(' '),
+            )
+            .status()
+            .expect("could not delete Kubernetes resources");
+        if !status.success() {
+            eprintln!("could not delete Kubernetes resources")
+        }
     }
 }
diff --git a/turbolift_internals/src/local_queue.rs b/turbolift_internals/src/local_queue.rs
index d4ff2001..af0eeaf7 100644
--- a/turbolift_internals/src/local_queue.rs
+++ b/turbolift_internals/src/local_queue.rs
@@ -26,6 +26,7 @@ pub struct LocalQueue {
     fn_name_to_process: HashMap<FunctionName, Child>,
     fn_name_to_binary_path: HashMap<FunctionName, std::path::PathBuf>,
     request_client: reqwest::Client,
+    run_id: Uuid,
 }
 
 impl LocalQueue {
@@ -38,12 +39,7 @@ impl LocalQueue {
 impl DistributionPlatform for LocalQueue {
     /// declare a function. Runs once.
     #[tracing::instrument(skip(project_tar))]
-    async fn declare(
-        &mut self,
-        function_name: &str,
-        run_id: Uuid,
-        project_tar: &[u8],
-    ) -> DistributionResult<()> {
+    async fn declare(&mut self, function_name: &str, project_tar: &[u8]) -> DistributionResult<()> {
         let relative_build_dir = Path::new(".")
             .join(".turbolift")
             .join(".worker_build_cache");
@@ -53,7 +49,7 @@ impl DistributionPlatform for LocalQueue {
         let function_executable = Path::new(CACHE_PATH.as_os_str()).join(format!(
             "{}_{}_server",
             function_name.to_string(),
-            run_id.as_u128()
+            self.run_id.as_u128()
         ));
         make_executable(&build_dir.join(function_name), Some(&function_executable))?;
         self.fn_name_to_binary_path
diff --git a/turbolift_macros/src/lib.rs b/turbolift_macros/src/lib.rs
index f8a29f6a..a931a337 100644
--- a/turbolift_macros/src/lib.rs
+++ b/turbolift_macros/src/lib.rs
@@ -213,7 +213,7 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS
 
             if !platform.has_declared(#original_target_function_name) {
                 platform
-                    .declare(#original_target_function_name, Uuid::new_v4(), #project_source_binary)
+                    .declare(#original_target_function_name, #project_source_binary)
                     .compat()
                     .await?;
             }

From 90c43ecd038bdfbd4b63561cd11ad6ebe8b0cdce Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sun, 13 Jun 2021 15:17:44 +0200
Subject: [PATCH 141/145] add todo to readme

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 5cf6856f..9d3398d1 100644
--- a/README.md
+++ b/README.md
@@ -85,3 +85,4 @@ don't build any microservices or alter any code.
 - [ ] refactor split between turbolift_internals and turbolift_macros
 - [ ] improve names
 - [ ] send params in json as payload instead of directly in the url
+- [ ] we need to do a better job of cleaning up docker images, locally and in the cluster.

From 6a63d09afcd6e7234e62bcb797d31730cf49aacf Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sun, 13 Jun 2021 15:28:15 +0200
Subject: [PATCH 142/145] update example text

---
 examples/README.md | 26 +++++++++++++++-----------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/examples/README.md b/examples/README.md
index 7a68bee2..7240d414 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -2,17 +2,21 @@
 
 ## Kubernetes
 
-Distributing portions of a rust application on Kubernetes was the reason turbolift was made. Internally, turbolift 
-uses the derived source code for each image to create a containerized HTTP server. The container is added to a local 
- registry, and a deployment and service is then created that exposes the server to requests from the main program. When 
- the main program completes and the K8s turbolift manager is dropped from memory, it removes the local registry with 
- the container, as well as the deployment and service. 
+Builds a deployment, service, and ingress rule for the function to be
+distributed. Cool features: distribution-as-a-feature, automatically deleting
+the pods, deployments, services, and ingress rule when the main program
+completes. This implementation is BYOC (Bring Your Own Container, you have to
+pass a special function while instantiating the cluster interface that allows
+makes the containers available in the cluster, perhaps via a private registry).
 
 ## Local Queue
 
-The local queue example should never be used in a production application. It's designed to test the core features of 
-turbolift (automatically extracting microservices from a rust codebase and running them on an http server),
-without any of the platform-specific code for e.g. running on kubernetes. Check this example out if you're interested in 
-a bare-bones example turbolift project without any platform-specific specialization. Note: if you're looking to run code 
-locally in turbolift instead of using a distribution platform, you should deactivate the distributed turbolift feature 
-in your project's `Cargo.toml`. This will let your program run all services locally, e.g. while developing.
\ No newline at end of file
+The local queue example should never be used in a production application. It's
+designed to test the core features of turbolift (automatically extracting
+microservices from a rust codebase and running them on an http server), without
+any of the platform-specific code for e.g. running on kubernetes. Check this
+example out if you're interested in a bare-bones example turbolift project
+without any platform-specific specialization. Note: if you're looking to run
+code locally in turbolift instead of using a distribution platform, you should
+deactivate the distributed turbolift feature in your project's `Cargo.toml`.
+This will let your program run all services locally, e.g. while developing.

From 1238ab7051cfbb429ba2c560722c0c36de7cf9a9 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sun, 13 Jun 2021 15:32:52 +0200
Subject: [PATCH 143/145] update todos

---
 README.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/README.md b/README.md
index 9d3398d1..4507cf47 100644
--- a/README.md
+++ b/README.md
@@ -72,6 +72,9 @@ e.g. pin the env or match the OS of the current environment.
 ## Current Project Goals
 
 - [X] support kubernetes ([pr](https://github.com/DominicBurkart/turbolift/pull/2)).
+- [ ] implement liveliness and readiness checks for pods.
+- [ ] while setting up a new service, wait for the pod to come alive via 
+readiness check instead of just sleeping ([code location](https://github.com/DominicBurkart/turbolift/blob/6a63d09afcd6e7234e62bcb797d31730cf49aacf/turbolift_internals/src/kubernetes.rs#L257)).
 - [ ] roadmap support for other targets.
 - [X] only use distributed configuration when flagged (like in
 `cargo build --features "distributed"`). Otherwise, just transform the

From 916349400b2c51ab2fd7339812ee33541e576ad5 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sun, 13 Jun 2021 15:41:49 +0200
Subject: [PATCH 144/145] add keywords and categories

---
 Cargo.toml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Cargo.toml b/Cargo.toml
index 0003934c..4fb98372 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -4,6 +4,8 @@ version = "0.1.1"
 authors = ["Dominic Burkart <@DominicBurkart>"]
 edition = "2018"
 description = "Easy distribution interface"
+keywords = ["distribution", "distributed", "kubernetes", "K8s"]
+categories = ["build-utils", "development-tools", "concurrency", "network-programming", "asynchronous"]
 readme = "README.md"
 homepage = "https://dominic.computer/turbolift"
 license = "Hippocratic-2.1"

From 1d418e758ddf233003d6075ef5d4adcf436fb6d2 Mon Sep 17 00:00:00 2001
From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com>
Date: Sun, 13 Jun 2021 16:04:47 +0200
Subject: [PATCH 145/145] add website badge to readme

---
 Cargo.toml | 1 +
 README.md  | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/Cargo.toml b/Cargo.toml
index 4fb98372..e215a250 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -9,6 +9,7 @@ categories = ["build-utils", "development-tools", "concurrency", "network-progra
 readme = "README.md"
 homepage = "https://dominic.computer/turbolift"
 license = "Hippocratic-2.1"
+repository = "https://github.com/dominicburkart/turbolift/"
 
 [features]
 distributed = ["chrono", "turbolift_macros/distributed"]
diff --git a/README.md b/README.md
index 4507cf47..d3b0b69d 100644
--- a/README.md
+++ b/README.md
@@ -5,6 +5,7 @@
     alt="turbolift’s current version badge"
     title="turbolift’s current version badge" />
 [![status](https://img.shields.io/github/checks-status/dominicburkart/turbolift/master)](https://github.com/DominicBurkart/turbolift/actions?query=branch%3Amaster)
+[![website](https://img.shields.io/badge/-website-blue)](https://dominic.computer/turbolift)
 
 Turbolift is a WIP distribution platform for rust. It's designed to make
 distribution an afterthought by extracting and distributing specific
@@ -73,7 +74,7 @@ e.g. pin the env or match the OS of the current environment.
 
 - [X] support kubernetes ([pr](https://github.com/DominicBurkart/turbolift/pull/2)).
 - [ ] implement liveliness and readiness checks for pods.
-- [ ] while setting up a new service, wait for the pod to come alive via 
+- [ ] while setting up a new service, wait for the pod to come alive via
 readiness check instead of just sleeping ([code location](https://github.com/DominicBurkart/turbolift/blob/6a63d09afcd6e7234e62bcb797d31730cf49aacf/turbolift_internals/src/kubernetes.rs#L257)).
 - [ ] roadmap support for other targets.
 - [X] only use distributed configuration when flagged (like in