diff --git a/README.md b/README.md index 291447d..d8c9630 100644 --- a/README.md +++ b/README.md @@ -12,9 +12,7 @@ Kubernetes offers the facility of extending it's API through the concept of 'Operators' ([Introducing Operators: Putting Operational Knowledge into Software](https://coreos.com/blog/introducing-operators.html)). This repository contains the resources and code to deploy an Azure Databricks Operator for Kubernetes. -The Azure Databricks operator comprises two projects: -- The golang application is a Kubernetes controller that watches Customer Resource Definitions (CRDs) that define a Databricks job, and, -- A Python Flask App which sends commands to the Databricks clusters. +It is a Kubernetes controller that watches Customer Resource Definitions (CRDs) that define a Databricks job. ![alt text](docs/images/azure-databricks-operator.jpg "high level architecture") @@ -23,9 +21,7 @@ The Databricks operator is useful in situations where Kubernetes hosted applicat The project was built using 1. [Kubebuilder](https://book.kubebuilder.io/) -2. [Swagger Codegen](https://github.com/swagger-api/swagger-codegen) -3. [Flask-RESTPlus](http://flask-restplus.readthedocs.io) -4. [Flask](http://flask.pocoo.org/) +2. [ Golang SDK for Azure DataBricks REST API 2.0](https://github.com/xinsnake/databricks-sdk-golang) ![alt text](docs/images/development-flow.jpg "development flow") @@ -38,8 +34,6 @@ The project was built using * To configure a local Kubernetes cluster on your machine > You need to make sure a kubeconfig file is configured. -3. [kustomize](https://github.com/kubernetes-sigs/kustomize) is also needed - Basic commands to check your cluster ```shell @@ -146,7 +140,7 @@ kubectl -n databricks-operator-system logs databricks-operator-controller-manage ### Run Souce Code -1. Clone the repo +1. Clone the repo - make sure your go path points to `microsoft\azure-databricks-operator` 2. Install the NotebookJob CRD in the configured Kubernetes cluster folder ~/.kube/config, run `kubectl apply -f databricks-operator/config/crds` or `make install -C databricks-operator` @@ -171,11 +165,11 @@ run `kubectl apply -f databricks-operator/config/crds` or `make install -C datab #### Updating the Databricks operator: -This Repo is generated by [Kubebuilder](https://book.kubebuilder.io/). +This Repo is generated by [Kubebuilder](https://book.kubebuilder.io/) `legacy version`. To Extend the operator `databricks-operator`: -1. Run `dep ensure` to download dependencies. It doesn't show any progress bar and takes a while to download all of dependencies. +1. Run `go mod tidy` to download dependencies. It doesn't show any progress bar and takes a while to download all of dependencies. 2. Update `pkg\apis\microsoft\v1beta1\notebookjob_types.go`. 3. Regenerate CRD `make manifests`. 4. Install updated CRD `make install` @@ -196,10 +190,11 @@ To Extend the operator `databricks-operator`: 2. Paul Bouwer [Github](https://github.com/paulbouwer), [Linkedin](https://www.linkedin.com/in/pbouwer/) 3. Lace Lofranco [Github](https://github.com/devlace), [Linkedin](https://www.linkedin.com/in/lacelofranco/) 4. Allan Targino [Github](https://github.com/allantargino), [Linkedin](https://www.linkedin.com/in/allan-targino//) -5. Jason Goodselli [Github](https://github.com/JasonTheDeveloper), [Linkedin](https://www.linkedin.com/in/jason-goodsell-2505a3b2/) -6. Craig Rodger [Github](https://github.com/crrodger), [Linkedin](https://www.linkedin.com/in/craigrodger/) -7. Justin Chizer [Github](https://github.com/justinchizer), [Linkedin](https://www.linkedin.com/in/jchizer/) -8. Azadeh Khojandi [Github](https://github.com/Azadehkhojandi), [Linkedin](https://www.linkedin.com/in/azadeh-khojandi-ba441b3/) +6. Xinyun (Jacob) Zhou[Github](https://github.com/xinsnake),[Linkedin](https://www.linkedin.com/in/xinyun-zhou/) +7. Jason Goodsell [Github](https://github.com/JasonTheDeveloper), [Linkedin](https://www.linkedin.com/in/jason-goodsell-2505a3b2/) +8. Craig Rodger [Github](https://github.com/crrodger), [Linkedin](https://www.linkedin.com/in/craigrodger/) +9. Justin Chizer [Github](https://github.com/justinchizer), [Linkedin](https://www.linkedin.com/in/jchizer/) +10. Azadeh Khojandi [Github](https://github.com/Azadehkhojandi), [Linkedin](https://www.linkedin.com/in/azadeh-khojandi-ba441b3/) ## Resources diff --git a/docs/images/azure-databricks-operator.jpg b/docs/images/azure-databricks-operator.jpg index 1c2da74..368cd26 100644 Binary files a/docs/images/azure-databricks-operator.jpg and b/docs/images/azure-databricks-operator.jpg differ diff --git a/docs/images/development-flow.jpg b/docs/images/development-flow.jpg index 616f2d3..b8eef2e 100644 Binary files a/docs/images/development-flow.jpg and b/docs/images/development-flow.jpg differ diff --git a/go.mod b/go.mod index b15873a..638fa6d 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/prometheus/procfs v0.0.0-20190403104016-ea9eea638872 // indirect github.com/spf13/pflag v1.0.3 // indirect github.com/stretchr/testify v1.3.0 // indirect - github.com/xinsnake/databricks-sdk-golang v0.0.0-20190621002826-727cae3171aa + github.com/xinsnake/databricks-sdk-golang v0.0.0-20190625012826-3a272d083f0b go.uber.org/atomic v1.3.2 // indirect go.uber.org/multierr v1.1.0 // indirect go.uber.org/zap v1.9.1 // indirect diff --git a/go.sum b/go.sum index 54471c2..d831550 100644 --- a/go.sum +++ b/go.sum @@ -134,6 +134,8 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/xinsnake/databricks-sdk-golang v0.0.0-20190621002826-727cae3171aa h1:UWYgt2m8WtGq8pk6NpPwk2GEo9o9Hep8TCbhbdJ324s= github.com/xinsnake/databricks-sdk-golang v0.0.0-20190621002826-727cae3171aa/go.mod h1:8RROKFtXYmb3l4nmJeWWlHmMRYh+V6ha0ugrMSxm2A0= +github.com/xinsnake/databricks-sdk-golang v0.0.0-20190625012826-3a272d083f0b h1:SJtDvKEaBPPiAAhSDgzPYUvbQ7Pmb5AIwNR6eWRxnk0= +github.com/xinsnake/databricks-sdk-golang v0.0.0-20190625012826-3a272d083f0b/go.mod h1:8RROKFtXYmb3l4nmJeWWlHmMRYh+V6ha0ugrMSxm2A0= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= diff --git a/pkg/apis/microsoft/v1beta1/zz_generated.deepcopy.go b/pkg/apis/microsoft/v1beta1/zz_generated.deepcopy.go index 791153d..e2ee6fe 100644 --- a/pkg/apis/microsoft/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/microsoft/v1beta1/zz_generated.deepcopy.go @@ -20,6 +20,7 @@ limitations under the License. package v1beta1 import ( + models "github.com/xinsnake/databricks-sdk-golang/azure/models" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -178,6 +179,17 @@ func (in *NotebookJobSpec) DeepCopy() *NotebookJobSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NotebookJobStatus) DeepCopyInto(out *NotebookJobStatus) { + *out = *in + if in.Run != nil { + in, out := &in.Run, &out.Run + *out = new(models.Run) + (*in).DeepCopyInto(*out) + } + return +} + // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NotebookJobStatus. func (in *NotebookJobStatus) DeepCopy() *NotebookJobStatus { if in == nil { diff --git a/pkg/apis/microsoft/v1beta1/zz_manual.deepcopy.go b/pkg/apis/microsoft/v1beta1/zz_manual.deepcopy.go deleted file mode 100644 index 83027d8..0000000 --- a/pkg/apis/microsoft/v1beta1/zz_manual.deepcopy.go +++ /dev/null @@ -1,6 +0,0 @@ -package v1beta1 - -func (in *NotebookJobStatus) DeepCopyInto(out *NotebookJobStatus) { - *out = *in - return -}