Porch

git: handle resolution of per-package versions (kptdev#2681)

We can have a git tag that looks like `packageDir/version`, this
allows for independent versioning of packages that share a git repo.

We need to try both ref forms when resolving git packages.

PackageDraft interface should not inherit from Package (kptdev#2678)

In particular, some of the methods such as GetResources are ambiguous
until we finish the draft (do we want the in-progress resources or the
upstream resources?)

Minor fixups to better surface errors (kptdev#2680)

In general, we should always return the error, unless we actually want
to mask it e.g. for security reasons (and in that case, we would
normally expect to log the underlying error instead of wrapping).

Clean up Makefile (kptdev#2685)

* Remove init and deinit (no longer useful)
* Fix module discovery
* Clean up directory references

Render and Eval via Simple Memory FS (kptdev#2684)

* Render and Eval via Simple Memory FS
  For now the filesystem is inserted at lower level; In future changes
  the mutation Apply will accept the filesystem abstraction to avoid
  back-and-forth between filesystem and PackageResources.
* Clean up Function Evaluation to use new interface
* Remove unnecessary kpt module

Working towards applying packages from a repository (kptdev#2686)

First steps (with lots of hard-coding and hacks) towards applying from
a repository to a cluster.

Add priority and fairness to RBAC roles (kptdev#2692)

aggregated-apiservers seem to query these resources, and spam the logs
(at least) if they don't have permission.

Applying: publish MVP status (kptdev#2695)

We start to expose the status of the sync operation.

Fix Docker Build (kptdev#2696)

Remove kpt directory from Docker build.

Tidy Go Modules (kptdev#2699)

* Fix Go Modules

* Run Tidy in Porch Workflow

Add Roundtrip Tests (kptdev#2703)

.github/workflows/porch.yml

@@ -41,3 +41,5 @@ jobs:
       - name: Test
         run: make test
         working-directory: ./porch
+      - name: Tidy
+        run: make tidy

porch/Makefile

@@ -12,6 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+KUBECONFIG=$(CURDIR)/hack/local/kubeconfig
+BUILDDIR=$(CURDIR)/.build
+CACHEDIR=$(CURDIR)/.cache
+MODULES = $(shell find . -path ./.build -prune -o -path ./.cache -prune -o -name 'go.mod' -print)
+
 .PHONY: all
 all: stop network start-etcd start-kube-apiserver run-local
 
@@ -30,13 +35,13 @@ stop:
 .PHONY: start-etcd
 start-etcd:
 	docker buildx build -t etcd --output=type=docker hack/local/etcd
-	mkdir -p .build/data/etcd
+	mkdir -p $(BUILDDIR)/data/etcd
 	docker stop etcd || true
 	docker rm etcd || true
 	docker run --detach --user `id -u`:`id -g` \
 	  --network=porch \
 	  --ip 192.168.8.200 \
-	  --name etcd -v `pwd`/.build/data/etcd:/data \
+	  --name etcd -v $(BUILDDIR)/data/etcd:/data \
 	  etcd --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://127.0.0.1:2379
 
 .PHONY: start-kube-apiserver
@@ -48,7 +53,7 @@ start-kube-apiserver:
 	docker run --detach --user `id -u`:`id -g` \
 	  --network=porch \
 	  --ip 192.168.8.201 \
-	  --name kube-apiserver -v `pwd`/.build/pki:/pki \
+	  --name kube-apiserver -v $(BUILDDIR)/pki:/pki \
 	  --add-host host.docker.internal:host-gateway \
 	  kube-apiserver \
 	  --etcd-servers http://etcd:2379 \
@@ -61,8 +66,6 @@ start-kube-apiserver:
 	  --anonymous-auth=false \
 	  --client-ca-file=/pki/ca.crt
 
-MODULES = $(shell find . -path ./forks -prune -path ./.build -prune -o -name 'go.mod' -print)
-
 .PHONY: generate
 generate: $(MODULES)
 	@for f in $(^D); do (cd $$f; echo "Generating $$f"; go generate -v ./...) || exit 1; done
@@ -84,24 +87,15 @@ vet: $(MODULES)
 fmt: $(MODULES)
 	@for f in $(^D); do (cd $$f; echo "Formatting $$f"; gofmt -s -w .); done
 
-KUBECONFIG=$(dir $(abspath $(lastword $(MAKEFILE_LIST))))hack/local/kubeconfig
-
-.PHONY: init deinit
-init:
-	@ [[ -z $$(find "./config/$${USER}" -type f -name '*.yaml') ]] || KUBECONFIG="$(KUBECONFIG)" kubectl apply -f "./config/$${USER}"
-
-deinit:
-	@ [[ -z $$(find "./config/$${USER}" -type f -name '*.yaml') ]] || KUBECONFIG="$(KUBECONFIG)" kubectl delete -f "./config/$${USER}"
-
 .PHONY: run-local
 run-local:
 	KUBECONFIG=$(KUBECONFIG) kubectl apply -f hack/local/localconfig.yaml
 	KUBECONFIG=$(KUBECONFIG) kubectl apply -f controllers/pkg/apis/porch/v1alpha1/
 	cd apiserver; go run ./cmd/porch \
 	--secure-port 9443 \
 	--standalone-debug-mode \
-	--kubeconfig=$(KUBECONFIG) \
-	--cache-directory="$(dir $(abspath $(lastword $(MAKEFILE_LIST)))).cache"
+	--kubeconfig="$(KUBECONFIG)" \
+	--cache-directory="$(CACHEDIR)"
 
 .PHONY: run-jaeger
 run-jaeger:

porch/README.md

@@ -161,7 +161,7 @@ kubectl api-resources
 kubectl get packagerevisions --namespace default
 ```
 
-Follow the instructions above on how to register repositories and discover packges.
+Follow the instructions above on how to register repositories and discover packages.
 
 ### Running Locally
 

porch/api/go.mod

@@ -6,6 +6,12 @@ go 1.17
 
 require k8s.io/apimachinery v0.23.0
 
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	google.golang.org/protobuf v1.27.1 // indirect
+)
+
 require (
 	github.com/PuerkitoBio/purell v1.1.1 // indirect
 	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect

porch/api/go.sum

@@ -51,6 +51,7 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -220,6 +221,7 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
 google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

porch/api/porch/fuzzer/fuzzer.go

@@ -0,0 +1,9 @@
+package fuzzer
+
+import (
+	runtimeserializer "k8s.io/apimachinery/pkg/runtime/serializer"
+)
+
+var Funcs = func(codecs runtimeserializer.CodecFactory) []interface{} {
+	return []interface{}{}
+}

porch/api/porch/install/roundtrip_test.go

@@ -0,0 +1,12 @@
+package install
+
+import (
+	"testing"
+
+	"github.com/GoogleContainerTools/kpt/porch/api/porch/fuzzer"
+	"k8s.io/apimachinery/pkg/api/apitesting/roundtrip"
+)
+
+func TestRoundTripTypes(t *testing.T) {
+	roundtrip.RoundTripTestForAPIGroup(t, Install, fuzzer.Funcs)
+}

porch/apiserver/go.mod

@@ -33,14 +33,12 @@ replace (
 	github.com/GoogleContainerTools/kpt/porch/apiserver => ./
 	github.com/GoogleContainerTools/kpt/porch/controllers => ../controllers
 	github.com/GoogleContainerTools/kpt/porch/engine => ../engine
-	github.com/GoogleContainerTools/kpt/porch/kpt => ../kpt
 	github.com/GoogleContainerTools/kpt/porch/repository => ../repository
 )
 
 require (
 	cloud.google.com/go v0.99.0 // indirect
 	github.com/GoogleContainerTools/kpt v0.0.0-00010101000000-000000000000 // indirect
-	github.com/GoogleContainerTools/kpt/porch/kpt v0.0.0-00010101000000-000000000000 // indirect
 	github.com/Microsoft/go-winio v0.5.1 // indirect
 	github.com/NYTimes/gziphandler v1.1.1 // indirect
 	github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 // indirect

porch/apiserver/pkg/apiserver/scheme_test.go

@@ -16,7 +16,11 @@ package apiserver
 
 import (
 	"testing"
+
+	"github.com/GoogleContainerTools/kpt/porch/api/porch/fuzzer"
+	"k8s.io/apimachinery/pkg/api/apitesting/roundtrip"
 )
 
 func TestRoundTripTypes(t *testing.T) {
+	roundtrip.RoundTripTestForScheme(t, Scheme, fuzzer.Funcs)
 }

porch/config/deploy/5-rbac.yaml

@@ -26,3 +26,7 @@ rules:
 - apiGroups: ["config.porch.kpt.dev"]
   resources: ["repositories"]
   verbs: ["get", "list", "watch", "create", "update", "patch"]
+# Needed for priority and fairness
+- apiGroups: ["flowcontrol.apiserver.k8s.io"]
+  resources: ["flowschemas", "prioritylevelconfigurations"]
+  verbs: ["get", "watch", "list"]

porch/config/samples/create-deployment-package.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+kubectl apply -f - <<EOF
+apiVersion: porch.kpt.dev/v1alpha1
+kind: PackageRevision
+metadata:
+  namespace: default
+  name: "deployment:myfirstnginx:v1"
+spec:
+  packageName: myfirstnginx
+  revision: v1
+  repository: deployment
+  tasks:
+  - type: clone
+    clone:
+      upstreamRef:
+        type: git
+        git:
+          repo: https://github.com/GoogleContainerTools/kpt
+          ref: v0.7
+          directory: package-examples/nginx
+EOF
+
+kubectl get packagerevision -n default deployment:myfirstnginx:v1 -oyaml
+
+kubectl get packagerevisionresources -n default deployment:myfirstnginx:v1 -oyaml
+

porch/config/samples/deployment-repository.yaml

@@ -0,0 +1,27 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: config.porch.kpt.dev/v1alpha1
+kind: Repository
+metadata:
+  name: deployment
+  namespace: default
+spec:
+  title: Deployment
+  description: Deployment Repository for testing Porch.
+  content: PackageRevision # TODO: Deployment or similar?
+  type: oci
+  oci:
+    # Replace with your OCI repository.
+    registry: us-west1-docker.pkg.dev/example-google-project-id/deployment

porch/controllers/remoterootsync/Makefile

@@ -0,0 +1,6 @@
+# GCP project to use for development
+GCP_PROJECT_ID ?= $(shell gcloud config get-value project)
+
+.PHONY: run-local
+run-local:
+	GCP_PROJECT_ID=${GCP_PROJECT_ID} HACK_ENABLE_LOOPBACK=1 go run .

porch/controllers/remoterootsync/api/v1alpha1/remoterootsyncset_types.go

@@ -67,6 +67,11 @@ type OCISpec struct {
 
 // RootSyncSetStatus defines the observed state of RootSyncSet
 type RemoteRootSyncSetStatus struct {
+	Targets []TargetStatus `json:"targets,omitempty"`
+}
+
+type TargetStatus struct {
+	Ref ClusterRef `json:"ref,omitempty"`
 	// Conditions describes the reconciliation state of the object.
 	Conditions []metav1.Condition `json:"conditions,omitempty"`
 }