Merge pull request #2 from DominicBurkart/kubernetes

kubernetes support

.github/workflows/examples.yml

@@ -0,0 +1,79 @@
+name: examples
+
+on:
+  push:
+    branches: master
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  local_queue_example:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: build local queue example
+        run: docker build -f examples/local_queue_example/Dockerfile .
+
+  kubernetes_example_local_test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          path: './turbolift'
+      - name: install rustup and rust nightly
+        run: |
+            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly-2020-09-28
+      - name: run tests
+        run: |
+            cd turbolift/examples/kubernetes_example
+            RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo test -- --nocapture
+
+  kubernetes_example_local_run:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          path: './turbolift'
+      - name: install rustup and rust nightly
+        run: |
+          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly-2020-09-28
+      - name: run tests
+        run: |
+          cd turbolift/examples/kubernetes_example
+          RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run
+
+  kubernetes_example_distributed_test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          path: './turbolift'
+      - name: install rustup and rust nightly
+        run: |
+          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly-2020-09-28
+      - uses: engineerd/[email protected]
+        with:
+          version: "v0.11.0"
+      - name: run tests
+        run: |
+          cd turbolift/examples/kubernetes_example
+          sh setup_cluster.sh
+          RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo test --features distributed -- --nocapture
+
+  kubernetes_example_distributed_run:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          path: './turbolift'
+      - name: install rustup and rust nightly
+        run: |
+          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly-2020-09-28
+      - uses: engineerd/[email protected]
+        with:
+          version: "v0.11.0"
+      - name: run tests
+        run: |
+          cd turbolift/examples/kubernetes_example
+          sh setup_cluster.sh
+          RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo run --features distributed

.github/workflows/rust.yml → .github/workflows/linting.yml

@@ -1,16 +1,16 @@
-name: rust
+name: linting
 
 on:
   push:
-    branches: [ master ]
+    branches: master
   pull_request:
     branches: [ master ]
 
 env:
   CARGO_TERM_COLOR: always
 
 jobs:
-  test:
+  lints:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
@@ -22,15 +22,15 @@ jobs:
           override: true
     - name: Rustup
       run: rustup update
-    - name: Format
+    - name: Format turbolift
       run: RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo fmt
+    - name: Format turbolift_internals
+      run: cd turbolift_internals && RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo fmt
+    - name: Format turbolift_macros
+      run: cd turbolift_macros && RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo
     - name: Check without distributed feature
       run: RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo check
     - name: Check with distributed feature
-      run: RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo check --features "distributed"
+      run: RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo check --features distributed
     - name: Clippy
       run: RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo clippy -- -D warnings
-    - name: Local Queue without distributed feature
-      run: cd examples/local_queue_example && RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo test
-    - name: Local Queue with distributed feature
-      run: cd examples/local_queue_example && RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo test --features "distributed"

.gitignore

@@ -1,4 +1,5 @@
 /target
 target/
 Cargo.lock
-.turbolift/
+.turbolift/
+.idea

.pre-commit-config.yaml

@@ -1,3 +1,4 @@
+repos:
 -    repo: local
      hooks:
         -   id: format
@@ -16,16 +17,31 @@
             types: [rust]
             pass_filenames: false
         -   id: check
-            name: Check
+            name: Cargo Check (local)
             description: Runs `cargo check` on the repository.
             entry: bash -c 'RUSTFLAGS='"'"'--cfg procmacro2_semver_exempt'"'"' cargo check "$@"' --
             language: system
             types: [ rust ]
             pass_filenames: false
         -   id: check
-            name: Check
+            name: Cargo Check (distributed)
             description: Runs `cargo check` on the repository with distributed flag
-            entry: bash -c 'RUSTFLAGS='"'"'--cfg procmacro2_semver_exempt'"'"' cargo check --features "distributed" "$@"' --
+            entry: bash -c 'RUSTFLAGS='"'"'--cfg procmacro2_semver_exempt'"'"' cargo check --features distributed "$@"' --
             language: system
             types: [ rust ]
-            pass_filenames: false
+            pass_filenames: false
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v3.2.0
+  hooks:
+    - id: trailing-whitespace
+    - id: mixed-line-ending
+    - id: end-of-file-fixer
+    - id: detect-private-key
+    - id: check-merge-conflict
+    - id: check-toml
+    - id: check-yaml
+- repo: https://github.com/jumanjihouse/pre-commit-hooks
+  rev: 2.1.4
+  hooks:
+    - id: markdownlint
+    - id: shellcheck

Cargo.toml

@@ -1,22 +1,26 @@
 [package]
 name = "turbolift"
-version = "0.1.0"
+version = "0.1.1"
 authors = ["Dominic Burkart <@DominicBurkart>"]
 edition = "2018"
-description = "WIP"
+description = "Easy distribution interface"
+keywords = ["distribution", "distributed", "kubernetes", "K8s"]
+categories = ["build-utils", "development-tools", "concurrency", "network-programming", "asynchronous"]
 readme = "README.md"
+homepage = "https://dominic.computer/turbolift"
 license = "Hippocratic-2.1"
+repository = "https://github.com/dominicburkart/turbolift/"
 
 [features]
-distributed = ["chrono", "cached", "async-std"]
-service = ["actix-web", "serde_json"]
+distributed = ["chrono", "turbolift_macros/distributed"]
 # todo we can optimize reqs for children with this load
 
 [dependencies]
 turbolift_macros = { path = "./turbolift_macros" }
 turbolift_internals = { path = "./turbolift_internals" }
-async-std = { version = "1.6", optional = true }
 chrono = { version = "0.4", optional = true }
-cached = { version = "0.19", optional = true }
-actix-web = { version = "3", optional = true }
-serde_json = { version = "1", optional = true }
+actix-web = { version = "3" }
+serde_json = { version = "1" }
+tokio-compat-02 = { version = "0.1" }
+tracing = {version="0.1", features=["attributes"]}
+tracing-futures = "0.2.4"

README.md

@@ -4,64 +4,89 @@
     src="https://img.shields.io/crates/v/turbolift.svg"
     alt="turbolift’s current version badge"
     title="turbolift’s current version badge" />
-[![status](https://github.com/DominicBurkart/turbolift/workflows/rust/badge.svg)](https://github.com/DominicBurkart/turbolift/actions?query=is%3Acompleted+branch%3Amaster+workflow%3A"rust")
-[![status](https://github.com/DominicBurkart/turbolift/workflows/docker/badge.svg)](https://github.com/DominicBurkart/turbolift/actions?query=is%3Acompleted+branch%3Amaster+workflow%3A"docker")
+[![status](https://img.shields.io/github/checks-status/dominicburkart/turbolift/master)](https://github.com/DominicBurkart/turbolift/actions?query=branch%3Amaster)
+[![website](https://img.shields.io/badge/-website-blue)](https://dominic.computer/turbolift)
 
-Turbolift is a WIP distribution platform for rust. It's designed to make distribution an afterthought 
-by extracting and distributing specific functions and their dependencies from a larger rust application.
-Turbolift then acts as the glue between these extracted mini-apps and the main application.
+Turbolift is a WIP distribution platform for rust. It's designed to make
+distribution an afterthought by extracting and distributing specific
+functions and their dependencies from a larger rust application.
+Turbolift then acts as the glue between these extracted mini-apps and
+the main application.
 
-Look in the [examples](https://github.com/DominicBurkart/turbolift/tree/master/examples) directory for 
-full projects with working syntax examples. 
+Look in the [examples](https://github.com/DominicBurkart/turbolift/tree/master/examples)
+directory for full projects with working syntax examples.
 
-## Distribution as an afterthought.
-Turbolift allows developers to turn normal rust functions into distributed services 
- just by tagging them with a macro. This lets you develop in a monorepo environment, 
-but benefit from the scalability of microservice architectures.
+## Distribution as an afterthought
 
-## Orchestration with a feature flag.
-For quicker development builds, `cargo build` doesn't build the distributed version of your code by default.
-Distribution is feature-gated so that it's easy to turn on (for example, in production), or off (for example, 
-while developing locally).
+Turbolift allows developers to turn normal rust functions into distributed services
+just by tagging them with a macro. Right now, Turbolift only works with K8s, though
+it's designed to be extended to other cluster management utilities.
+
+## Orchestration with a feature flag
+
+Distribution is feature-gated in Turbolift, so it's easy to activate distribution
+for some builds and deactivate it for others (while developing locally, for
+example).
 
 ## Important implementation notes
-- implemented over http using `reqwest` and `actix-web` (no current plans to refactor to use a lower level network protocol).
-- assumes a secure network– function parameters are sent in plaintext to the microservice.
-- source vulnerability: when building, anything in the project directory or in local dependencies 
-declared in the project manifest could be bundled and sent over the network to workers. 
+
+- implemented over http using `reqwest` and `actix-web` (no current plans to
+refactor to use a lower level network protocol).
+- assumes a secure network– function parameters are sent in plaintext to the
+microservice.
+- source vulnerability: when building, anything in the project directory or in
+local dependencies declared in the project manifest could be bundled and sent
+over the network to workers.
+
+More information is available on the [project homepage](https://dominic.computer/turbolift).
 
 ## Current Limitations
-- *Because of reliance on unstable proc_macro::Span features, all programs using turbolift need to 
-be built with an unstable nightly compiler flag (e.g. `RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo build`)* ([tracking issue](https://github.com/rust-lang/rust/issues/54725)).
-- Functions are assumed to be pure (lacking side-effects such as 
-writing to the file system or mutation of a global variable). 
-*Today, this is not enforced by the code.* 
-- For a function to be distributed, its inputs and outputs have to be (de)serializable with [Serde](https://github.com/serde-rs/serde).
+
+- DO NOT RUN TURBOLIFT ON A PUBLIC-FACING CLUSTER.
+- *Because of reliance on unstable proc_macro::Span features, all programs
+using turbolift need to be built with an unstable nightly compiler flag (e.g.
+`RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo build`)*
+([tracking issue](https://github.com/rust-lang/rust/issues/54725)).
+- Functions are assumed to be pure (lacking side-effects such as
+writing to the file system or mutation of a global variable).
+*Today, this is not enforced by the code.*
+- For a function to be distributed, its inputs and outputs have to be
+(de)serializable with [Serde](https://github.com/serde-rs/serde).
 - Distributed functions cannot be nested in other functions.
 - Distributed functions cannot be methods.
 - Distributed functions cannot use other functions called `main`.
-- Distributed functions not in `main.rs` cannot use functions declared 
+- Distributed functions not in `main.rs` cannot use functions declared
 in `main.rs`.
 - Distributed functions cannot have `-> impl Trait` types.
-- Unused functions that have been marked with the `on` macro will still be 
-compiled for distribution, even if eventually the linker will then 
+- Unused functions that have been marked with the `on` macro will still be
+compiled for distribution, even if eventually the linker will then
 remove the completed binary and distribution code.
-- *Turbolift doesn't match the cargo compilation settings for microservices yet.*
-- projects can have relative local dependencies listing in the cargo manifest, but those dependencies themselves 
-should not have relative local dependencies prone to breaking.
-- if your program produces side effects when initialized, for example when 
-global constants are initialized, those side effects may be triggered 
+- projects can have relative local dependencies listing in the cargo
+manifest, but those dependencies themselves should not have relative local
+dependencies prone to breaking.
+- if your program produces side effects when initialized, for example when
+global constants are initialized, those side effects may be triggered
 for each function call.
+- turbolift runs functions on an unreproducible linux build, it doesn't
+e.g. pin the env or match the OS of the current environment.
 
 ## Current Project Goals
-- [ ] support kubernetes ([pr](https://github.com/DominicBurkart/turbolift/pull/2)).
+
+- [X] support kubernetes ([pr](https://github.com/DominicBurkart/turbolift/pull/2)).
+- [ ] implement liveliness and readiness checks for pods.
+- [ ] while setting up a new service, wait for the pod to come alive via
+readiness check instead of just sleeping ([code location](https://github.com/DominicBurkart/turbolift/blob/6a63d09afcd6e7234e62bcb797d31730cf49aacf/turbolift_internals/src/kubernetes.rs#L257)).
 - [ ] roadmap support for other targets.
-- [X] only use distributed configuration when flagged (like in `cargo build --features "distributed"`). Otherwise,
-just transform the tagged function into an async function (to provide an identical API), but don't 
-build any microservices or alter any code.
-- [ ] build cross-architecture compilation tests into the CI (RN we only test via github actions read Docker, and a different custom Docker test workflow)
+- [X] only use distributed configuration when flagged (like in
+`cargo build --features "distributed"`). Otherwise, just transform the
+tagged function into an async function (to provide an identical API), but
+don't build any microservices or alter any code.
+- [ ] build cross-architecture compilation tests into the CI.
 
 ## Current tech debt todo
+
 - [ ] start reducing ginormous API, right now basically everything is public
 - [ ] refactor split between turbolift_internals and turbolift_macros
 - [ ] improve names
+- [ ] send params in json as payload instead of directly in the url
+- [ ] we need to do a better job of cleaning up docker images, locally and in the cluster.

examples/README.md

@@ -0,0 +1,22 @@
+# Turbolift Examples
+
+## Kubernetes
+
+Builds a deployment, service, and ingress rule for the function to be
+distributed. Cool features: distribution-as-a-feature, automatically deleting
+the pods, deployments, services, and ingress rule when the main program
+completes. This implementation is BYOC (Bring Your Own Container, you have to
+pass a special function while instantiating the cluster interface that allows
+makes the containers available in the cluster, perhaps via a private registry).
+
+## Local Queue
+
+The local queue example should never be used in a production application. It's
+designed to test the core features of turbolift (automatically extracting
+microservices from a rust codebase and running them on an http server), without
+any of the platform-specific code for e.g. running on kubernetes. Check this
+example out if you're interested in a bare-bones example turbolift project
+without any platform-specific specialization. Note: if you're looking to run
+code locally in turbolift instead of using a distribution platform, you should
+deactivate the distributed turbolift feature in your project's `Cargo.toml`.
+This will let your program run all services locally, e.g. while developing.

examples/kubernetes_example/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "kubernetes_example"
+version = "0.1.0"
+authors = ["Dominic Burkart <[email protected]>"]
+edition = "2018"
+
+[features]
+"distributed" = ["turbolift/distributed"]
+
+[dependencies]
+rand = "0.7"
+tokio = {version="1", features=["full"]}
+lazy_static = "1"
+futures = "0.3"
+cute = "0.3"
+anyhow = "1.0.41"
+turbolift = { path="../../" }
+
+# for printing out tracing
+tracing = "0.1"
+tracing-futures = "0.2"
+tracing-subscriber = {version="0.2", features=["fmt"]}