diff --git a/porch/Makefile b/porch/Makefile index 7e95d934a6..743e7daf95 100644 --- a/porch/Makefile +++ b/porch/Makefile @@ -17,6 +17,9 @@ BUILDDIR=$(CURDIR)/.build CACHEDIR=$(CURDIR)/.cache MODULES = $(shell find . -path ./.build -prune -o -path ./.cache -prune -o -name 'go.mod' -print) +# GCP project to use for development +GCP_PROJECT_ID ?= $(shell gcloud config get-value project) + .PHONY: all all: stop network start-etcd start-kube-apiserver start-function-runner run-local @@ -125,3 +128,8 @@ fix-headers: .PHONY: fix-all fix-all: fix-headers fmt tidy + +.PHONY: push-images +push-images: + hack/build-image.sh --project $(GCP_PROJECT_ID) --push + make -C controllers/ push-image diff --git a/porch/apiserver/go.mod b/porch/apiserver/go.mod index a075eaa681..68215e1cdb 100644 --- a/porch/apiserver/go.mod +++ b/porch/apiserver/go.mod @@ -15,7 +15,7 @@ require ( go.opentelemetry.io/otel/exporters/stdout v0.20.0 go.opentelemetry.io/otel/sdk v0.20.0 go.opentelemetry.io/otel/sdk/metric v0.20.0 - google.golang.org/grpc v1.43.0 + google.golang.org/grpc v1.44.0 k8s.io/api v0.23.1 k8s.io/apimachinery v0.23.1 k8s.io/apiserver v0.23.0 @@ -33,6 +33,7 @@ replace ( github.com/GoogleContainerTools/kpt/porch/apiserver => ./ github.com/GoogleContainerTools/kpt/porch/controllers => ../controllers github.com/GoogleContainerTools/kpt/porch/engine => ../engine + github.com/GoogleContainerTools/kpt/porch/func => ../func github.com/GoogleContainerTools/kpt/porch/repository => ../repository ) @@ -40,6 +41,7 @@ require ( cloud.google.com/go v0.99.0 // indirect github.com/GoogleContainerTools/kpt v0.0.0-00010101000000-000000000000 // indirect github.com/GoogleContainerTools/kpt-functions-catalog/functions/go/apply-setters v0.2.0 // indirect + github.com/GoogleContainerTools/kpt/porch/func v0.0.0-00010101000000-000000000000 // indirect github.com/Microsoft/go-winio v0.5.1 // indirect github.com/NYTimes/gziphandler v1.1.1 // indirect github.com/ProtonMail/go-crypto v0.0.0-20220113124808-70ae35bab23f // indirect @@ -76,6 +78,7 @@ require ( github.com/google/go-cmp v0.5.7 // indirect github.com/google/go-containerregistry v0.8.0 // indirect github.com/google/gofuzz v1.1.0 // indirect + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/uuid v1.3.0 // indirect github.com/googleapis/gnostic v0.5.5 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect @@ -122,6 +125,7 @@ require ( go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.19.1 // indirect golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect + golang.org/x/mod v0.5.1 // indirect golang.org/x/net v0.0.0-20211216030914-fe4d6282115f // indirect golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect diff --git a/porch/apiserver/go.sum b/porch/apiserver/go.sum index 03cc188d28..add5ca50a9 100644 --- a/porch/apiserver/go.sum +++ b/porch/apiserver/go.sum @@ -1558,8 +1558,9 @@ google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.43.0 h1:Eeu7bZtDZ2DpRCsLhUlcrLnvYaMK1Gz86a+hMVvELmM= google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.44.0 h1:weqSxi/TMs1SqFRMHCtBgXRs8k3X39QIDEZ0pRcttUg= +google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= diff --git a/porch/apiserver/pkg/apiserver/apiserver.go b/porch/apiserver/pkg/apiserver/apiserver.go index 61dd957be5..af2fa0d9a0 100644 --- a/porch/apiserver/pkg/apiserver/apiserver.go +++ b/porch/apiserver/pkg/apiserver/apiserver.go @@ -62,6 +62,7 @@ func init() { type ExtraConfig struct { CoreAPIKubeconfigPath string CacheDirectory string + FunctionRunnerAddress string } // Config defines the config for the apiserver @@ -159,7 +160,9 @@ func (c completedConfig) New() (*PorchServer, error) { return nil, fmt.Errorf("failed to build client for core apiserver: %w", err) } - porchGroup, err := porch.NewRESTStorage(Scheme, Codecs, c.GenericConfig.RESTOptionsGetter, coreClient, c.ExtraConfig.CacheDirectory) + porchGroup, err := porch.NewRESTStorage( + Scheme, Codecs, c.GenericConfig.RESTOptionsGetter, coreClient, + c.ExtraConfig.CacheDirectory, c.ExtraConfig.FunctionRunnerAddress) if err != nil { return nil, err } diff --git a/porch/apiserver/pkg/cmd/server/start.go b/porch/apiserver/pkg/cmd/server/start.go index 0b0319508e..90901009d6 100644 --- a/porch/apiserver/pkg/cmd/server/start.go +++ b/porch/apiserver/pkg/cmd/server/start.go @@ -48,6 +48,7 @@ type PorchServerOptions struct { LocalStandaloneDebugging bool // Enables local standalone running/debugging of the apiserver. CacheDirectory string CoreAPIKubeconfigPath string + FunctionRunnerAddress string SharedInformerFactory informers.SharedInformerFactory StdOut io.Writer @@ -179,6 +180,7 @@ func (o *PorchServerOptions) Config() (*apiserver.Config, error) { ExtraConfig: apiserver.ExtraConfig{ CoreAPIKubeconfigPath: o.CoreAPIKubeconfigPath, CacheDirectory: o.CacheDirectory, + FunctionRunnerAddress: o.FunctionRunnerAddress, }, } return config, nil @@ -221,5 +223,6 @@ func (o *PorchServerOptions) AddFlags(fs *pflag.FlagSet) { "authorizing the requests, this flag is only intended for debugging in your workstation.") } + fs.StringVar(&o.FunctionRunnerAddress, "function-runner", "", "Address of the function runner gRPC service.") fs.StringVar(&o.CacheDirectory, "cache-directory", "", "Directory where Porch server stores repository and package caches.") } diff --git a/porch/apiserver/pkg/registry/porch/storage.go b/porch/apiserver/pkg/registry/porch/storage.go index ed409463c6..dcd4964cf0 100644 --- a/porch/apiserver/pkg/registry/porch/storage.go +++ b/porch/apiserver/pkg/registry/porch/storage.go @@ -30,10 +30,10 @@ import ( ) func NewRESTStorage(scheme *runtime.Scheme, codecs serializer.CodecFactory, restOptionsGetter genericregistry.RESTOptionsGetter, - coreClient client.WithWatch, cacheDirectory string) (genericapiserver.APIGroupInfo, error) { + coreClient client.WithWatch, cacheDirectory string, functionRunnerAddress string) (genericapiserver.APIGroupInfo, error) { c := cache.NewCache(cacheDirectory) - cad, err := engine.NewCaDEngine(c) + cad, err := engine.NewCaDEngine(c, functionRunnerAddress) if err != nil { return genericapiserver.APIGroupInfo{}, err diff --git a/porch/controllers/Dockerfile b/porch/controllers/Dockerfile new file mode 100644 index 0000000000..201fdfe2eb --- /dev/null +++ b/porch/controllers/Dockerfile @@ -0,0 +1,54 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM golang:1.17-bullseye as builder + +WORKDIR /workspace +COPY go.mod go.sum ./ +COPY porch/api/go.mod porch/api/go.sum porch/api/ +COPY porch/controllers/go.mod porch/controllers/go.sum porch/controllers/ +COPY porch/controllers/remoterootsync/go.mod porch/controllers/remoterootsync/go.sum porch/controllers/remoterootsync/ +COPY porch/repository/go.mod porch/repository/go.sum porch/repository/ + +WORKDIR /workspace/porch/controllers/remoterootsync/ +RUN go mod download +# Prebuild some libraries to warm the cache +RUN CGO_ENABLED=0 go build -v \ + k8s.io/klog/v2 \ + k8s.io/klog/v2/klogr \ + k8s.io/client-go/plugin/pkg/client/auth \ + sigs.k8s.io/controller-runtime \ + sigs.k8s.io/controller-runtime/pkg/client \ + sigs.k8s.io/controller-runtime/pkg/controller/controllerutil \ + k8s.io/client-go/kubernetes \ + go.opentelemetry.io/otel \ + cloud.google.com/go/container/apiv1 \ + github.com/google/go-containerregistry/pkg/gcrane \ + github.com/google/go-containerregistry/pkg/v1 \ + github.com/google/go-containerregistry/pkg/v1/cache \ + k8s.io/client-go/discovery/cached + +WORKDIR /workspace +COPY porch/api/ porch/api/ +COPY porch/controllers/ porch/controllers/ +COPY porch/repository/ porch/repository/ + +WORKDIR /workspace/porch/controllers/remoterootsync/ +RUN CGO_ENABLED=0 go build -o /porch-controllers -v . + +FROM gcr.io/distroless/static +WORKDIR /data +COPY --from=builder /porch-controllers /porch-controllers + +ENTRYPOINT ["/porch-controllers"] diff --git a/porch/controllers/Makefile b/porch/controllers/Makefile new file mode 100644 index 0000000000..5709021495 --- /dev/null +++ b/porch/controllers/Makefile @@ -0,0 +1,20 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# GCP project to use for development +GCP_PROJECT_ID ?= $(shell gcloud config get-value project) + +.PHONY: push-image +push-image: + cd ../..; docker buildx build --push --tag gcr.io/${GCP_PROJECT_ID}/porch-controllers:latest -f porch/controllers/Dockerfile . diff --git a/porch/controllers/config/deploy/manifest.yaml b/porch/controllers/config/deploy/manifest.yaml new file mode 100644 index 0000000000..1c5b96d935 --- /dev/null +++ b/porch/controllers/config/deploy/manifest.yaml @@ -0,0 +1,83 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Namespace +metadata: + name: porch-system + +--- + +kind: ServiceAccount +apiVersion: v1 +metadata: + name: porch-controllers + namespace: porch-system + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: porch-controllers + namespace: porch-system + labels: + k8s-app: "porch-controllers" +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: "porch-controllers" + template: + metadata: + labels: + k8s-app: "porch-controllers" + spec: + serviceAccountName: porch-controllers + containers: + - name: porch-controllers + # Update to the image of your porch-controllers build. + image: gcr.io/example-google-project-id/porch-controllers:latest + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: porch-controllers +rules: +- apiGroups: ["config.porch.kpt.dev"] + resources: ["repositories"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +- apiGroups: ["config.cloud.google.com"] + resources: ["remoterootsyncsets"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +- apiGroups: ["config.cloud.google.com"] + resources: ["remoterootsyncsets/status"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: porch-system:porch-controllers +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: porch-controllers +subjects: +- kind: ServiceAccount + name: porch-controllers + namespace: porch-system diff --git a/porch/controllers/remoterootsync/Makefile b/porch/controllers/remoterootsync/Makefile index 3bef1be8bb..d33ce97e22 100644 --- a/porch/controllers/remoterootsync/Makefile +++ b/porch/controllers/remoterootsync/Makefile @@ -17,4 +17,4 @@ GCP_PROJECT_ID ?= $(shell gcloud config get-value project) .PHONY: run-local run-local: - GCP_PROJECT_ID=${GCP_PROJECT_ID} HACK_ENABLE_LOOPBACK=1 go run . \ No newline at end of file + GCP_PROJECT_ID=${GCP_PROJECT_ID} HACK_ENABLE_LOOPBACK=1 go run . diff --git a/porch/controllers/remoterootsync/api/v1alpha1/remoterootsyncset_types.go b/porch/controllers/remoterootsync/api/v1alpha1/remoterootsyncset_types.go index 8e9f55c807..85bae86bc0 100644 --- a/porch/controllers/remoterootsync/api/v1alpha1/remoterootsyncset_types.go +++ b/porch/controllers/remoterootsync/api/v1alpha1/remoterootsyncset_types.go @@ -42,12 +42,26 @@ type RemoteRootSyncSet struct { Status RemoteRootSyncSetStatus `json:"status,omitempty"` } +func (o *RemoteRootSyncSet) GetSpec() *RemoteRootSyncSetSpec { + if o == nil { + return nil + } + return &o.Spec +} + // RemoteRootSyncSetSpec defines the desired state of RemoteRootSync type RemoteRootSyncSetSpec struct { ClusterRefs []*ClusterRef `json:"clusterRefs,omitempty"` Template *RootSyncTemplate `json:"template,omitempty"` } +func (o *RemoteRootSyncSetSpec) GetTemplate() *RootSyncTemplate { + if o == nil { + return nil + } + return o.Template +} + type ClusterRef struct { ApiVersion string `json:"apiVersion,omitempty"` Kind string `json:"kind,omitempty"` @@ -61,10 +75,24 @@ type RootSyncTemplate struct { OCI *OCISpec `json:"oci,omitempty"` } +func (o *RootSyncTemplate) GetOCI() *OCISpec { + if o == nil { + return nil + } + return o.OCI +} + type OCISpec struct { Repository string `json:"repository,omitempty"` } +func (o *OCISpec) GetRepository() string { + if o == nil { + return "" + } + return o.Repository +} + // RootSyncSetStatus defines the observed state of RootSyncSet type RemoteRootSyncSetStatus struct { Targets []TargetStatus `json:"targets,omitempty"` diff --git a/porch/controllers/remoterootsync/pkg/controllers/remoterootsyncset/remoterootsync_controller.go b/porch/controllers/remoterootsync/pkg/controllers/remoterootsyncset/remoterootsync_controller.go index 291ef74eee..f767f4cbb5 100644 --- a/porch/controllers/remoterootsync/pkg/controllers/remoterootsyncset/remoterootsync_controller.go +++ b/porch/controllers/remoterootsync/pkg/controllers/remoterootsyncset/remoterootsync_controller.go @@ -207,7 +207,15 @@ func (r *RemoteRootSyncSetReconciler) applyToClusterRef(ctx context.Context, sub } // TODO: Cache applyset - patchOptions := metav1.PatchOptions{FieldManager: "remoterootsync-" + subject.GetNamespace() + "-" + subject.GetName()} + patchOptions := metav1.PatchOptions{ + FieldManager: "remoterootsync-" + subject.GetNamespace() + "-" + subject.GetName(), + } + + // We force to overcome errors like: Apply failed with 1 conflict: conflict with "kubectl-client-side-apply" using apps/v1: .spec.template.spec.containers[name="porch-server"].image + // TODO: How to handle this better + force := true + patchOptions.Force = &force + applyset, err := applyset.New(applyset.Options{ RESTMapper: restMapper, Client: client, @@ -233,14 +241,17 @@ func (r *RemoteRootSyncSetReconciler) applyToClusterRef(ctx context.Context, sub // BuildObjectsToApply config root sync func (r *RemoteRootSyncSetReconciler) BuildObjectsToApply(ctx context.Context, subject *api.RemoteRootSyncSet) ([]*unstructured.Unstructured, error) { - // TODO: stop hard-coding the image source; get from a deployment instead - gcpProjectID := os.Getenv("GCP_PROJECT_ID") - imageName := oci.ImageTagName{ - Image: "us-west1-docker.pkg.dev/" + gcpProjectID + "/deployment/myfirstnginx", - Tag: "v1", + repository := subject.GetSpec().GetTemplate().GetOCI().GetRepository() + if repository == "" { + return nil, fmt.Errorf("spec.template.oci.repository is not set") } + imageName, err := oci.ParseImageTagName(repository) + if err != nil { + return nil, fmt.Errorf("unable to parse image %q: %w", repository, err) + } + klog.Infof("image name %s -> %#v", repository, *imageName) - digest, err := r.ociStorage.LookupImageTag(ctx, imageName) + digest, err := r.ociStorage.LookupImageTag(ctx, *imageName) if err != nil { return nil, err } @@ -279,12 +290,6 @@ func (r *RemoteRootSyncSetReconciler) BuildObjectsToApply(ctx context.Context, s return nil, fmt.Errorf("error parsing yaml from %s: %w", item.Path, err) } - // Hack: default namespace until we populate the namespace in our packages - if o.GetNamespace() == "" { - klog.Warningf("HACK: setting namespace to default") - o.SetNamespace("default") - } - // TODO: sync with kpt logic; skip objects marked with the local-only annotation objects = append(objects, o) } diff --git a/porch/engine/go.mod b/porch/engine/go.mod index be00a9467a..a8efa0b874 100644 --- a/porch/engine/go.mod +++ b/porch/engine/go.mod @@ -7,8 +7,10 @@ require ( github.com/GoogleContainerTools/kpt-functions-catalog/functions/go/apply-setters v0.2.0 github.com/GoogleContainerTools/kpt/porch/api v0.0.0-00010101000000-000000000000 github.com/GoogleContainerTools/kpt/porch/controllers v0.0.0-00010101000000-000000000000 + github.com/GoogleContainerTools/kpt/porch/func v0.0.0-00010101000000-000000000000 github.com/GoogleContainerTools/kpt/porch/repository v0.0.0-00010101000000-000000000000 github.com/google/go-cmp v0.5.7 + google.golang.org/grpc v1.44.0 k8s.io/klog/v2 v2.40.1 sigs.k8s.io/kustomize/kyaml v0.13.3 ) @@ -17,6 +19,7 @@ replace ( github.com/GoogleContainerTools/kpt => ../../ github.com/GoogleContainerTools/kpt/porch/api => ../api github.com/GoogleContainerTools/kpt/porch/controllers => ../controllers + github.com/GoogleContainerTools/kpt/porch/func => ../func github.com/GoogleContainerTools/kpt/porch/kpt => ../kpt github.com/GoogleContainerTools/kpt/porch/repository => ../repository ) @@ -84,6 +87,7 @@ require ( golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect golang.org/x/text v0.3.7 // indirect google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect google.golang.org/protobuf v1.27.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect diff --git a/porch/engine/go.sum b/porch/engine/go.sum index d440543351..95c5b1949c 100644 --- a/porch/engine/go.sum +++ b/porch/engine/go.sum @@ -1447,6 +1447,7 @@ google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ6 google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211203200212-54befc351ae9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa h1:I0YcKz0I7OAhddo7ya8kMnvprhcWM045PmkBdMO9zN0= google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -1480,6 +1481,8 @@ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.44.0 h1:weqSxi/TMs1SqFRMHCtBgXRs8k3X39QIDEZ0pRcttUg= +google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= diff --git a/porch/engine/pkg/engine/engine.go b/porch/engine/pkg/engine/engine.go index 167ea25b9b..11c43f7f88 100644 --- a/porch/engine/pkg/engine/engine.go +++ b/porch/engine/pkg/engine/engine.go @@ -30,8 +30,12 @@ import ( api "github.com/GoogleContainerTools/kpt/porch/api/porch/v1alpha1" configapi "github.com/GoogleContainerTools/kpt/porch/controllers/pkg/apis/porch/v1alpha1" "github.com/GoogleContainerTools/kpt/porch/engine/pkg/kpt" + "github.com/GoogleContainerTools/kpt/porch/func/evaluator" "github.com/GoogleContainerTools/kpt/porch/repository/pkg/cache" "github.com/GoogleContainerTools/kpt/porch/repository/pkg/repository" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "k8s.io/klog/v2" "sigs.k8s.io/kustomize/kyaml/kio" "sigs.k8s.io/kustomize/kyaml/yaml" ) @@ -45,14 +49,38 @@ type CaDEngine interface { ListFunctions(ctx context.Context, repositoryObj *configapi.Repository, auth repository.AuthOptions) ([]repository.Function, error) } -func NewCaDEngine(cache *cache.Cache) (CaDEngine, error) { +func NewCaDEngine(cache *cache.Cache, functionRunnerAddress string) (CaDEngine, error) { + runtime, err := createFunctionRuntime(functionRunnerAddress) + if err != nil { + return nil, fmt.Errorf("failed to create function runtime: %w", err) + } + return &cadEngine{ cache: cache, - renderer: kpt.NewPlaceholderRenderer(), - runtime: kpt.NewPlaceholderFunctionRuntime(), + renderer: kpt.NewRenderer(), + runtime: runtime, }, nil } +func createFunctionRuntime(address string) (kpt.FunctionRuntime, error) { + if address == "" { + klog.Warningf("Using simple kpt function runner (in-process)") + return kpt.NewSimpleFunctionRuntime(), nil + } + + klog.Infof("Dialing grpc function runner %q", address) + + cc, err := grpc.Dial(address, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + return nil, fmt.Errorf("failed to dial grpc function evaluator: %w", err) + } + + return &grpcRuntime{ + cc: cc, + client: evaluator.NewFunctionEvaluatorClient(cc), + }, err +} + type cadEngine struct { cache *cache.Cache renderer fn.Renderer @@ -82,35 +110,11 @@ func (cad *cadEngine) CreatePackageRevision(ctx context.Context, repositoryObj * var mutations []mutation for i := range obj.Spec.Tasks { task := &obj.Spec.Tasks[i] - switch task.Type { - case api.TaskTypeClone: - if task.Clone == nil { - return nil, fmt.Errorf("clone not set for task of type %q", task.Type) - } - mutations = append(mutations, &clonePackageMutation{ - task: task, - name: obj.Spec.PackageName, - }) - - case api.TaskTypePatch: - if task.Patch == nil { - return nil, fmt.Errorf("patch not set for task of type %q", task.Type) - } - // TODO: support patch? - return nil, fmt.Errorf("patch not supported on create") - - case api.TaskTypeEval: - if task.Eval == nil { - return nil, fmt.Errorf("eval not set for task of type %q", task.Type) - } - mutations = append(mutations, &evalFunctionMutation{ - runtime: cad.runtime, - task: task, - }) - - default: - return nil, fmt.Errorf("task of type %q not supported", task.Type) + mutation, err := cad.mapTaskToMutation(ctx, obj, task) + if err != nil { + return nil, err } + mutations = append(mutations, mutation) } // Render package after creation. @@ -124,6 +128,38 @@ func (cad *cadEngine) CreatePackageRevision(ctx context.Context, repositoryObj * return updateDraft(ctx, draft, baseResources, mutations) } +func (cad *cadEngine) mapTaskToMutation(ctx context.Context, obj *api.PackageRevision, task *api.Task) (mutation, error) { + switch task.Type { + case api.TaskTypeClone: + if task.Clone == nil { + return nil, fmt.Errorf("clone not set for task of type %q", task.Type) + } + return &clonePackageMutation{ + task: task, + name: obj.Spec.PackageName, + }, nil + + case api.TaskTypePatch: + if task.Patch == nil { + return nil, fmt.Errorf("patch not set for task of type %q", task.Type) + } + // TODO: support patch? + return nil, fmt.Errorf("patch not supported on create") + + case api.TaskTypeEval: + if task.Eval == nil { + return nil, fmt.Errorf("eval not set for task of type %q", task.Type) + } + return &evalFunctionMutation{ + runtime: cad.runtime, + task: task, + }, nil + + default: + return nil, fmt.Errorf("task of type %q not supported", task.Type) + } +} + func (cad *cadEngine) UpdatePackageRevision(ctx context.Context, repositoryObj *configapi.Repository, auth repository.AuthOptions, oldPackage repository.PackageRevision, oldObj, newObj *api.PackageRevision) (repository.PackageRevision, error) { repo, err := cad.cache.OpenRepository(repositoryObj, auth) if err != nil { diff --git a/porch/engine/pkg/engine/grpcruntime.go b/porch/engine/pkg/engine/grpcruntime.go new file mode 100644 index 0000000000..f9d63b1caf --- /dev/null +++ b/porch/engine/pkg/engine/grpcruntime.go @@ -0,0 +1,82 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package engine + +import ( + "context" + "fmt" + "io" + "io/ioutil" + + v1 "github.com/GoogleContainerTools/kpt/pkg/api/kptfile/v1" + "github.com/GoogleContainerTools/kpt/pkg/fn" + "github.com/GoogleContainerTools/kpt/porch/engine/pkg/kpt" + "github.com/GoogleContainerTools/kpt/porch/func/evaluator" + "google.golang.org/grpc" + "k8s.io/klog/v2" +) + +type grpcRuntime struct { + cc *grpc.ClientConn + client evaluator.FunctionEvaluatorClient +} + +var _ kpt.FunctionRuntime = &grpcRuntime{} + +func (gr *grpcRuntime) GetRunner(ctx context.Context, fn *v1.Function) (fn.FunctionRunner, error) { + return &grpcRunner{ + ctx: ctx, + client: gr.client, + image: fn.Image, + }, nil +} + +func (gr *grpcRuntime) Close() error { + var err error + if gr.cc != nil { + if err = gr.cc.Close(); err != nil { + klog.Warningf("Failed to close grpc client connection: %v", err) + } + gr.cc = nil + } + return err +} + +type grpcRunner struct { + ctx context.Context + client evaluator.FunctionEvaluatorClient + image string +} + +var _ fn.FunctionRunner = &grpcRunner{} + +func (gr *grpcRunner) Run(r io.Reader, w io.Writer) error { + in, err := ioutil.ReadAll(r) + if err != nil { + return fmt.Errorf("failed to read function runner input: %w", err) + } + + res, err := gr.client.EvaluateFunction(gr.ctx, &evaluator.EvaluateFunctionRequest{ + ResourceList: in, + Image: gr.image, + }) + if err != nil { + return fmt.Errorf("func eval failed: %w (%s)", err, string(res.Log)) + } + if _, err := w.Write(res.ResourceList); err != nil { + return fmt.Errorf("failed to write function runner output: %w", err) + } + return nil +} diff --git a/porch/engine/pkg/engine/render.go b/porch/engine/pkg/engine/render.go index 3f7624907f..347cda5460 100644 --- a/porch/engine/pkg/engine/render.go +++ b/porch/engine/pkg/engine/render.go @@ -36,11 +36,13 @@ var _ mutation = &renderPackageMutation{} func (m *renderPackageMutation) Apply(ctx context.Context, resources repository.PackageResources) (repository.PackageResources, *api.Task, error) { fs := filesys.MakeFsInMemory() - if err := writeResources(fs, resources); err != nil { + pkgPath, err := writeResources(fs, resources) + if err != nil { return repository.PackageResources{}, nil, err } if err := m.renderer.Render(ctx, fs, fn.RenderOptions{ + PkgPath: pkgPath, Runtime: m.runtime, }); err != nil { return repository.PackageResources{}, nil, err @@ -62,16 +64,30 @@ func (m *renderPackageMutation) Apply(ctx context.Context, resources repository. } // TODO: Implement filesystem abstraction directly rather than on top of PackageResources -func writeResources(fs filesys.FileSystem, resources repository.PackageResources) error { +func writeResources(fs filesys.FileSystem, resources repository.PackageResources) (string, error) { + var packageDir string // path to the topmost directory containing Kptfile for k, v := range resources.Contents { - if err := fs.MkdirAll(path.Dir(k)); err != nil { - return err + dir := path.Dir(k) + if dir == "." { + dir = "/" + } + if err := fs.MkdirAll(dir); err != nil { + return "", err } - if err := fs.WriteFile(k, []byte(v)); err != nil { - return err + base := path.Base(k) + if err := fs.WriteFile(path.Join(dir, base), []byte(v)); err != nil { + return "", err + } + if base == "Kptfile" { + // Found Kptfile. Check if the current directory is ancestor of the current + // topmost package directory. If so, use it instead. + if packageDir == "" || strings.HasPrefix(packageDir, dir+"/") { + packageDir = dir + } } } - return nil + // Return topmost directory containing Kptfile + return packageDir, nil } func readResources(fs filesys.FileSystem) (repository.PackageResources, error) { diff --git a/porch/engine/pkg/engine/render_test.go b/porch/engine/pkg/engine/render_test.go index 217838c614..4f0f755ef5 100644 --- a/porch/engine/pkg/engine/render_test.go +++ b/porch/engine/pkg/engine/render_test.go @@ -16,57 +16,62 @@ package engine import ( "context" + "io/ioutil" + "path/filepath" "testing" + v1 "github.com/GoogleContainerTools/kpt/pkg/api/kptfile/v1" "github.com/GoogleContainerTools/kpt/porch/engine/pkg/kpt" "github.com/GoogleContainerTools/kpt/porch/repository/pkg/repository" + "github.com/google/go-cmp/cmp" + "sigs.k8s.io/kustomize/kyaml/filesys" "sigs.k8s.io/kustomize/kyaml/kio" ) func TestRender(t *testing.T) { render := &renderPackageMutation{ - renderer: kpt.NewPlaceholderRenderer(), - runtime: kpt.NewPlaceholderFunctionRuntime(), + renderer: kpt.NewRenderer(), + runtime: kpt.NewSimpleFunctionRuntime(), } - const path = "bucket.yaml" - const annotation = "porch.kpt.dev/rendered" - const content = `# Comment -apiVersion: storage.cnrm.cloud.google.com/v1beta1 -kind: StorageBucket -metadata: - name: blueprints-project-bucket - namespace: config-control -spec: - storageClass: standard -` + testdata, err := filepath.Abs(filepath.Join(".", "testdata", "simple-render")) + if err != nil { + t.Fatalf("Failed to find testdata: %v", err) + } + packagePath := filepath.Join(testdata, "simple-bucket") + r := &kio.LocalPackageReader{ + PackagePath: packagePath, + IncludeSubpackages: true, + MatchFilesGlob: append(kio.MatchAll, v1.KptFileName), + FileSystem: filesys.FileSystemOrOnDisk{}, + } - resources := repository.PackageResources{ - Contents: map[string]string{ - path: content, + w := &packageWriter{ + output: repository.PackageResources{ + Contents: map[string]string{}, }, } - output, _, err := render.Apply(context.Background(), resources) + if err := (kio.Pipeline{Inputs: []kio.Reader{r}, Outputs: []kio.Writer{w}}).Execute(); err != nil { + t.Fatalf("Failed to read package: %v", err) + } + + rendered, _, err := render.Apply(context.Background(), w.output) if err != nil { t.Errorf("package render failed: %v", err) } - if got, want := len(output.Contents), 1; got != want { - t.Errorf("Expected single resource in the result. got %d", got) + got, ok := rendered.Contents["bucket.yaml"] + if !ok { + t.Errorf("Cannot find output config (bucket.yaml) in %v", rendered.Contents) } - result, err := kio.ParseAll(output.Contents[path]) + want, err := ioutil.ReadFile(filepath.Join(testdata, "expected.txt")) if err != nil { - t.Errorf("Failed to parse rendered package content: %v", err) + t.Fatalf("Cannot read expected.txt: %v", err) } - for _, n := range result { - annotations := n.GetAnnotations() - if got, ok := annotations[annotation]; !ok { - t.Errorf("expected %q annotation, got none", annotation) - } else if want := "yes"; got != want { - t.Errorf("%q annotation: got %q, want %q", annotation, got, want) - } + if diff := cmp.Diff(string(want), got); diff != "" { + t.Errorf("Unexpected result (-want, +got): %s", diff) } } diff --git a/porch/engine/pkg/engine/testdata/simple-render/expected.txt b/porch/engine/pkg/engine/testdata/simple-render/expected.txt new file mode 100644 index 0000000000..0e6c0ff9eb --- /dev/null +++ b/porch/engine/pkg/engine/testdata/simple-render/expected.txt @@ -0,0 +1,27 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: storage.cnrm.cloud.google.com/v1beta1 +kind: StorageBucket +metadata: # kpt-merge: config-control/blueprints-project-bucket + name: updated-project-id-updated-bucket-name # kpt-set: ${project-id}-${name} + namespace: updated-namespace # kpt-set: ${namespace} + annotations: + cnrm.cloud.google.com/force-destroy: "false" + cnrm.cloud.google.com/project-id: updated-project-id # kpt-set: ${project-id} + cnrm.cloud.google.com/blueprint: 'kpt-fn' +spec: + storageClass: updated-storage-class # kpt-set: ${storage-class} + uniformBucketLevelAccess: true + versioning: + enabled: false diff --git a/porch/engine/pkg/engine/testdata/simple-render/simple-bucket/Kptfile b/porch/engine/pkg/engine/testdata/simple-render/simple-bucket/Kptfile new file mode 100644 index 0000000000..cbda71fc5c --- /dev/null +++ b/porch/engine/pkg/engine/testdata/simple-render/simple-bucket/Kptfile @@ -0,0 +1,29 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: kpt.dev/v1 +kind: Kptfile +metadata: + name: simple-bucket + annotations: + blueprints.cloud.google.com/title: Google Cloud Storage Bucket blueprint +info: + description: A Google Cloud Storage bucket +pipeline: + mutators: + - image: gcr.io/kpt-fn/apply-setters:v0.2.0 + configMap: + name: updated-bucket-name + namespace: updated-namespace + project-id: updated-project-id + storage-class: updated-storage-class diff --git a/porch/engine/pkg/engine/testdata/simple-render/simple-bucket/bucket.yaml b/porch/engine/pkg/engine/testdata/simple-render/simple-bucket/bucket.yaml new file mode 100644 index 0000000000..551e893ca9 --- /dev/null +++ b/porch/engine/pkg/engine/testdata/simple-render/simple-bucket/bucket.yaml @@ -0,0 +1,26 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: storage.cnrm.cloud.google.com/v1beta1 +kind: StorageBucket +metadata: # kpt-merge: config-control/blueprints-project-bucket + name: blueprints-project-bucket # kpt-set: ${project-id}-${name} + namespace: config-control # kpt-set: ${namespace} + annotations: + cnrm.cloud.google.com/force-destroy: "false" + cnrm.cloud.google.com/project-id: blueprints-project # kpt-set: ${project-id} +spec: + storageClass: standard # kpt-set: ${storage-class} + uniformBucketLevelAccess: true + versioning: + enabled: false diff --git a/porch/engine/pkg/kpt/eval.go b/porch/engine/pkg/kpt/eval.go index c69332409c..52fa470407 100644 --- a/porch/engine/pkg/kpt/eval.go +++ b/porch/engine/pkg/kpt/eval.go @@ -26,14 +26,14 @@ import ( "sigs.k8s.io/kustomize/kyaml/kio" ) -func NewPlaceholderFunctionRuntime() fn.FunctionRuntime { +func NewSimpleFunctionRuntime() FunctionRuntime { return &runtime{} } type runtime struct { } -var _ fn.FunctionRuntime = &runtime{} +var _ FunctionRuntime = &runtime{} func (e *runtime) GetRunner(ctx context.Context, fn *kptfilev1.Function) (fn.FunctionRunner, error) { processor := internal.FindProcessor(fn.Image) @@ -48,6 +48,10 @@ func (e *runtime) GetRunner(ctx context.Context, fn *kptfilev1.Function) (fn.Fun }, nil } +func (e *runtime) Close() error { + return nil +} + type runner struct { ctx context.Context fn kptfilev1.Function diff --git a/porch/engine/pkg/kpt/fnruntime.go b/porch/engine/pkg/kpt/fnruntime.go new file mode 100644 index 0000000000..3bcdfab51f --- /dev/null +++ b/porch/engine/pkg/kpt/fnruntime.go @@ -0,0 +1,26 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kpt + +import ( + "io" + + "github.com/GoogleContainerTools/kpt/pkg/fn" +) + +type FunctionRuntime interface { + fn.FunctionRuntime + io.Closer +} diff --git a/porch/engine/pkg/kpt/render.go b/porch/engine/pkg/kpt/render.go index 6d17571360..7185275f0d 100644 --- a/porch/engine/pkg/kpt/render.go +++ b/porch/engine/pkg/kpt/render.go @@ -16,13 +16,19 @@ package kpt import ( "context" + "fmt" + "io" + "os" + "github.com/GoogleContainerTools/kpt/internal/pkg" + "github.com/GoogleContainerTools/kpt/internal/printer" + "github.com/GoogleContainerTools/kpt/internal/util/render" "github.com/GoogleContainerTools/kpt/pkg/fn" + "k8s.io/klog/v2" "sigs.k8s.io/kustomize/kyaml/filesys" - "sigs.k8s.io/kustomize/kyaml/kio" ) -func NewPlaceholderRenderer() fn.Renderer { +func NewRenderer() fn.Renderer { return &renderer{} } @@ -32,25 +38,45 @@ type renderer struct { var _ fn.Renderer = &renderer{} func (r *renderer) Render(ctx context.Context, pkg filesys.FileSystem, opts fn.RenderOptions) error { - rw := &kio.LocalPackageReadWriter{ - PackagePath: "/", - IncludeSubpackages: true, - FileSystem: filesys.FileSystemOrOnDisk{ - FileSystem: pkg, - }, + rr := render.Renderer{ + PkgPath: opts.PkgPath, + Runtime: opts.Runtime, + FileSystem: pkg, } - // Currently a noop rendering. TODO: Implement - nodes, err := rw.Read() - if err != nil { - return err - } + return rr.Execute(printer.WithContext(ctx, &packagePrinter{})) +} + +type packagePrinter struct{} + +var _ printer.Printer = &packagePrinter{} + +func (p *packagePrinter) PrintPackage(pkg *pkg.Pkg, leadingNewline bool) { + p.Printf("Package %q: ", pkg.DisplayPath) +} - for _, n := range nodes { - ann := n.GetAnnotations() - ann["porch.kpt.dev/rendered"] = "yes" - n.SetAnnotations(ann) +func (p *packagePrinter) Printf(format string, args ...interface{}) { + klog.Infof(format, args...) +} + +func (p *packagePrinter) OptPrintf(opt *printer.Options, format string, args ...interface{}) { + if opt == nil { + p.Printf(format, args...) + return + } + var prefix string + if !opt.PkgDisplayPath.Empty() { + prefix = fmt.Sprintf("Package %q: ", string(opt.PkgDisplayPath)) + } else if !opt.PkgPath.Empty() { + prefix = fmt.Sprintf("Package %q: ", string(opt.PkgPath)) } + p.Printf(prefix+format, args...) +} + +func (p *packagePrinter) OutStream() io.Writer { + return os.Stdout +} - return rw.Write(nodes) +func (p *packagePrinter) ErrStream() io.Writer { + return os.Stderr } diff --git a/porch/func/client/main.go b/porch/func/client/main.go index cf12fe7aca..e4feff2c8a 100644 --- a/porch/func/client/main.go +++ b/porch/func/client/main.go @@ -66,10 +66,7 @@ func createResourceList(args []string) ([]byte, error) { r := kio.LocalPackageReader{ PackagePath: *packageFlag, IncludeSubpackages: true, - } - nodes, err := r.Read() - if err != nil { - return nil, fmt.Errorf("failed to read package: %w", err) + WrapBareSeqNode: true, } cfg, err := configmap(args) @@ -77,9 +74,9 @@ func createResourceList(args []string) ([]byte, error) { return nil, fmt.Errorf("failed to create function config: %w", err) } - b := bytes.NewBuffer(nil) + var b bytes.Buffer w := kio.ByteWriter{ - Writer: b, + Writer: &b, KeepReaderAnnotations: true, Style: 0, FunctionConfig: cfg, @@ -87,7 +84,7 @@ func createResourceList(args []string) ([]byte, error) { WrappingAPIVersion: kio.ResourceListAPIVersion, } - if err := w.Write(nodes); err != nil { + if err := (kio.Pipeline{Inputs: []kio.Reader{r}, Outputs: []kio.Writer{w}}).Execute(); err != nil { return nil, fmt.Errorf("failed to create serialized ResourceList: %w", err) } diff --git a/porch/func/config/deploy/function-runner.yaml b/porch/func/config/deploy/function-runner.yaml new file mode 100644 index 0000000000..6bdbc83f91 --- /dev/null +++ b/porch/func/config/deploy/function-runner.yaml @@ -0,0 +1,50 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: function-runner + namespace: porch-system +spec: + replicas: 2 + selector: + matchLabels: + app: function-runner + template: + metadata: + labels: + app: function-runner + spec: + containers: + - name: function-runner + image: UPDATE_ME + imagePullPolicy: Always + ports: + - containerPort: 9445 + +--- + +apiVersion: v1 +kind: Service +metadata: + name: function-runner + namespace: porch-system +spec: + selector: + app: function-runner + ports: + - port: 9445 + protocol: TCP + targetPort: 9445 diff --git a/porch/func/config/Kptfile b/porch/func/config/sample/Kptfile similarity index 100% rename from porch/func/config/Kptfile rename to porch/func/config/sample/Kptfile diff --git a/porch/func/config/README.md b/porch/func/config/sample/README.md similarity index 100% rename from porch/func/config/README.md rename to porch/func/config/sample/README.md diff --git a/porch/func/config/configmap.yaml b/porch/func/config/sample/configmap.yaml similarity index 100% rename from porch/func/config/configmap.yaml rename to porch/func/config/sample/configmap.yaml diff --git a/porch/hack/Dockerfile b/porch/hack/Dockerfile index 4203a5a667..dfd227354a 100644 --- a/porch/hack/Dockerfile +++ b/porch/hack/Dockerfile @@ -26,6 +26,7 @@ COPY porch/apiserver/go.mod porch/apiserver/go.sum porch/apiserver/ COPY porch/controllers/go.mod porch/controllers/go.sum porch/controllers/ COPY porch/engine/go.mod porch/engine/go.mod porch/engine/ COPY porch/repository/go.mod porch/repository/go.sum porch/repository/ +COPY porch/func/go.mod porch/func/go.sum porch/func/ RUN echo "Downloading root modules ..." \ && go mod download @@ -39,12 +40,29 @@ RUN echo "Downloading engine modules ..." \ && cd porch/engine && go mod download RUN echo "Downloading repository modules ..." \ && cd porch/repository && go mod download +RUN echo "Downloading func modules ..." \ + && cd porch/func && go mod download + +# Prebuild some library dependencies to warm the cache +RUN cd porch/apiserver; go build -v \ + google.golang.org/grpc \ + k8s.io/apiserver/pkg/server \ + k8s.io/component-base/cli \ + k8s.io/klog/v2 \ + github.com/google/go-containerregistry/pkg/gcrane \ + k8s.io/client-go/kubernetes/scheme \ + github.com/go-git/go-git/v5 COPY internal internal COPY pkg pkg -COPY porch porch +COPY porch/api porch/api +COPY porch/apiserver porch/apiserver +COPY porch/controllers porch/controllers +COPY porch/engine porch/engine +COPY porch/repository porch/repository + -RUN cd porch/apiserver; go build -o /porch ./cmd/porch +RUN cd porch/apiserver; go build -v -o /porch ./cmd/porch FROM debian:bullseye RUN apt update && apt install -y ca-certificates && rm -rf /var/lib/apt && rm -rf /var/cache/apt diff --git a/porch/hack/build-image.sh b/porch/hack/build-image.sh index 0ee9442630..131ceed460 100755 --- a/porch/hack/build-image.sh +++ b/porch/hack/build-image.sh @@ -77,7 +77,7 @@ Usage: build-image.sh [flags] Supported Flags: --project [GCP_PROJECT] ... will build image gcr.io/{GCP_PROJECT}/porch:${TAG} --tag [TAG] ... tag for the image, i.e. 'latest' - --repository [REPOSITORY] ... the Doker image repository. will build image + --repository [REPOSITORY] ... the image repository. will build image [REPOSITORY]/porch:${TAG} --push ... push the image to the repository also EOF @@ -96,5 +96,5 @@ fi IMAGE="${REPOSITORY}/porch:${TAG}" -run docker build -t "${IMAGE}" -f "${BASE_DIR}/porch/hack/Dockerfile" "${BASE_DIR}" -[[ "${PUSH}" != "Yes" ]] || run docker push "${IMAGE}" +[[ "${PUSH}" == "Yes" ]] || run docker buildx build --load -t "${IMAGE}" -f "${BASE_DIR}/porch/hack/Dockerfile" "${BASE_DIR}" +[[ "${PUSH}" != "Yes" ]] || run docker buildx build --push -t "${IMAGE}" -f "${BASE_DIR}/porch/hack/Dockerfile" "${BASE_DIR}" diff --git a/porch/repository/pkg/oci/storage.go b/porch/repository/pkg/oci/storage.go index 003bcb8256..3c5766d378 100644 --- a/porch/repository/pkg/oci/storage.go +++ b/porch/repository/pkg/oci/storage.go @@ -71,6 +71,17 @@ func (i ImageTagName) ociReference() (name.Reference, error) { return imageRef, nil } +func ParseImageTagName(s string) (*ImageTagName, error) { + t, err := name.NewTag(s) + if err != nil { + return nil, fmt.Errorf("cannot parse %q as tag: %w", s, err) + } + return &ImageTagName{ + Image: t.Repository.Name(), + Tag: t.TagStr(), + }, nil +} + // ImageDigestName holds an image we know by digest (which is immutable and more cacheable) type ImageDigestName struct { Image string