Backport v1alpha1 hacks (#160)

* Upgraded storage version of kafkachannel (#446)

* Upgraded storage version of kafkachannel

Signed-off-by: Francesco Guardiani <[email protected]>

* Fix tests

Signed-off-by: Francesco Guardiani <[email protected]>

* Added post-install script to migrate KafkaChannel to v1beta1 (#448)

* Added post-install script to migrate KafkaChannel to v1beta1

Signed-off-by: Francesco Guardiani <[email protected]>

* ./hack/update-codegen.sh

Signed-off-by: Francesco Guardiani <[email protected]>

* Fixed labels

Signed-off-by: Francesco Guardiani <[email protected]>

* Fix labels stuff

Signed-off-by: Francesco Guardiani <[email protected]>

* Reverted also tests stuff

Signed-off-by: Francesco Guardiani <[email protected]>

* Tap subscriptions too when running the post nstal (#473)

Signed-off-by: Francesco Guardiani <[email protected]>

config/channel/resources/kafkachannel-crd.yaml

@@ -27,7 +27,7 @@ spec:
     - &version
       name: v1alpha1
       served: true
-      storage: true
+      storage: false
       subresources:
         status: { }
       schema:
@@ -54,7 +54,7 @@ spec:
     - <<: *version
       name: v1beta1
       served: true
-      storage: false
+      storage: true
       schema:
         openAPIV3Schema:
           <<: *openAPIV3Schema

config/post-install/clusterrole.yaml

@@ -0,0 +1,47 @@
+# Copyright 2021 The Knative Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: knative-kafka-eventing-post-install-job-role
+  labels:
+    kafka.eventing.knative.dev/release: devel
+rules:
+  # Storage version upgrader needs to be able to patch CRDs.
+  - apiGroups:
+      - "apiextensions.k8s.io"
+    resources:
+      - "customresourcedefinitions"
+      - "customresourcedefinitions/status"
+    verbs:
+      - "get"
+      - "list"
+      - "update"
+      - "patch"
+      - "watch"
+  # Our own resources we care about.
+  - apiGroups:
+      - "messaging.knative.dev"
+    resources:
+      - "kafkachannels"
+      - "subscriptions"
+    verbs:
+      - "get"
+      - "list"
+      - "create"
+      - "update"
+      - "delete"
+      - "patch"
+      - "watch"

config/post-install/placeholder.go

@@ -0,0 +1,19 @@
+/*
+Copyright 2021 The Knative Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package pre_install is a placeholder that allows us to pull in config files
+// via go mod vendor.
+package post_install

config/post-install/serviceaccount.yaml

@@ -0,0 +1,38 @@
+# Copyright 2021 The Knative Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: knative-kafka-eventing-post-install-job
+  namespace: knative-eventing
+  labels:
+    kafka.eventing.knative.dev/release: devel
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: knative-kafka-eventing-post-install-job-role-binding
+  labels:
+    kafka.eventing.knative.dev/release: devel
+subjects:
+  - kind: ServiceAccount
+    name: knative-kafka-eventing-post-install-job
+    namespace: knative-eventing
+roleRef:
+  kind: ClusterRole
+  name: knative-kafka-eventing-post-install-job-role
+  apiGroup: rbac.authorization.k8s.io

config/post-install/storage-version-migrator.yaml

@@ -0,0 +1,46 @@
+# Copyright 2021 The Knative Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: batch/v1
+kind: Job
+metadata:
+  # Adding a prefix to avoid naming conflict with previous version's post-install jobs,
+  # we cannot use `generateName` here as it's not supported by `kubectl apply -f`
+  #
+  # If `ttlSecondsAfterFinished` feature gate becomes generally available in the future,
+  # we can rely on that and keep using the same Job name.
+  name: v0.22-kafka-storage-version-migration
+  namespace: knative-eventing
+  labels:
+    app: "storage-version-migration"
+    kafka.eventing.knative.dev/release: devel
+spec:
+  ttlSecondsAfterFinished: 600
+  backoffLimit: 10
+  template:
+    metadata:
+      labels:
+        app: "storage-version-migration"
+        eventing.knative.dev/release: devel
+      annotations:
+        sidecar.istio.io/inject: "false"
+    spec:
+      serviceAccountName: knative-kafka-eventing-post-install-job
+      restartPolicy: OnFailure
+      containers:
+        - name: migrate
+          image: ko://knative.dev/eventing-kafka/vendor/knative.dev/pkg/apiextensions/storageversion/cmd/migrate
+          args:
+            - "kafkachannels.messaging.knative.dev"
+            - "subscriptions.messaging.knative.dev"

hack/release.sh

@@ -49,6 +49,15 @@ function build_release() {
     ko resolve ${KO_FLAGS} -f ${config}/ | "${LABEL_YAML_CMD[@]}" > ${yaml}
     all_yamls+=(${yaml})
   done
+
+  if [ -d "${YAML_REPO_ROOT}/config/post-install" ]; then
+    echo "Resolving post install manifests"
+
+    local yaml="post-install.yaml"
+    ko resolve ${KO_FLAGS} -f config/post-install/ | "${LABEL_YAML_CMD[@]}" > ${yaml}
+    all_yamls+=(${yaml})
+  fi
+
   ARTIFACTS_TO_PUBLISH="${all_yamls[@]}"
 }
 

hack/tools.go

@@ -32,4 +32,7 @@ import (
 	_ "knative.dev/eventing/test/test_images/wathola-forwarder"
 	_ "knative.dev/eventing/test/test_images/wathola-receiver"
 	_ "knative.dev/eventing/test/test_images/wathola-sender"
+
+	// For migration
+	_ "knative.dev/pkg/apiextensions/storageversion/cmd/migrate"
 )

pkg/apis/messaging/v1beta1/kafka_channel_defaults.go

@@ -32,9 +32,14 @@ func (c *KafkaChannel) SetDefaults(ctx context.Context) {
 	if c.Annotations == nil {
 		c.Annotations = make(map[string]string)
 	}
-	if _, ok := c.Annotations[messaging.SubscribableDuckVersionAnnotation]; !ok {
-		c.Annotations[messaging.SubscribableDuckVersionAnnotation] = "v1"
-	}
+
+	// TODO this should be wrapped in a if to avoid conversion of ducks and let some other duck controller do that?!
+	//   BUT we need this hack to properly update from v1alpha1 to v1beta1, so:
+	//   * KEEP THIS FOR THE WHOLE 0.22 LIFECYCLE
+	//   * REMOVE THIS BEFORE 0.23 RELEASE
+	//if _, ok := c.Annotations[messaging.SubscribableDuckVersionAnnotation]; !ok {
+	c.Annotations[messaging.SubscribableDuckVersionAnnotation] = "v1"
+	//}
 
 	c.Spec.SetDefaults(ctx)
 }

vendor/knative.dev/pkg/apiextensions/storageversion/cmd/migrate/config.go

@@ -0,0 +1,76 @@
+/*
+Copyright 2020 The Knative Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"os/user"
+	"path/filepath"
+
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+)
+
+// TODO - make a package that's reusable here and by sharedmain
+
+func configOrDie() *rest.Config {
+	var (
+		serverURL = flag.String("server", "",
+			"The address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster.")
+		kubeconfig = flag.String("kubeconfig", "",
+			"Path to a kubeconfig. Only required if out-of-cluster.")
+	)
+
+	flag.Parse()
+
+	cfg, err := getConfig(*serverURL, *kubeconfig)
+	if err != nil {
+		panic(fmt.Sprint("Error building kubeconfig:", err))
+	}
+
+	return cfg
+}
+
+// getConfig returns a rest.Config to be used for kubernetes client creation.
+// It does so in the following order:
+//   1. Use the passed kubeconfig/serverURL.
+//   2. Fallback to the KUBECONFIG environment variable.
+//   3. Fallback to in-cluster config.
+//   4. Fallback to the ~/.kube/config.
+func getConfig(serverURL, kubeconfig string) (*rest.Config, error) {
+	if kubeconfig == "" {
+		kubeconfig = os.Getenv("KUBECONFIG")
+	}
+	// If we have an explicit indication of where the kubernetes config lives, read that.
+	if kubeconfig != "" {
+		return clientcmd.BuildConfigFromFlags(serverURL, kubeconfig)
+	}
+	// If not, try the in-cluster config.
+	if c, err := rest.InClusterConfig(); err == nil {
+		return c, nil
+	}
+	// If no in-cluster config, try the default location in the user's home directory.
+	if usr, err := user.Current(); err == nil {
+		if c, err := clientcmd.BuildConfigFromFlags("", filepath.Join(usr.HomeDir, ".kube", "config")); err == nil {
+			return c, nil
+		}
+	}
+
+	return nil, fmt.Errorf("could not create a valid kubeconfig")
+}

vendor/knative.dev/pkg/apiextensions/storageversion/cmd/migrate/main.go

@@ -0,0 +1,84 @@
+/*
+Copyright 2020 The Knative Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+
+	"go.uber.org/zap"
+	apixclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/dynamic"
+	"knative.dev/pkg/apiextensions/storageversion"
+	"knative.dev/pkg/logging"
+	"knative.dev/pkg/signals"
+)
+
+func main() {
+	logger := setupLogger()
+	defer logger.Sync()
+
+	config := configOrDie()
+	grs, err := parseResources(flag.Args())
+	if err != nil {
+		logger.Fatal(err)
+	}
+
+	migrator := storageversion.NewMigrator(
+		dynamic.NewForConfigOrDie(config),
+		apixclient.NewForConfigOrDie(config),
+	)
+
+	ctx := signals.NewContext()
+
+	logger.Infof("Migrating %d group resources", len(grs))
+
+	for _, gr := range grs {
+		logger.Info("Migrating group resource ", gr)
+		if err := migrator.Migrate(ctx, gr); err != nil {
+			logger.Fatal("Failed to migrate: ", err)
+		}
+	}
+
+	logger.Info("Migration complete")
+}
+
+func parseResources(args []string) ([]schema.GroupResource, error) {
+	grs := make([]schema.GroupResource, 0, len(args))
+	for _, arg := range args {
+		gr := schema.ParseGroupResource(arg)
+		if gr.Empty() {
+			return nil, fmt.Errorf("unable to parse group version: %s", arg)
+		}
+		grs = append(grs, gr)
+	}
+	return grs, nil
+}
+
+func setupLogger() *zap.SugaredLogger {
+	const component = "storage-migrator"
+
+	config, err := logging.NewConfigFromMap(nil)
+	if err != nil {
+		log.Fatal("Failed to create logging config: ", err)
+	}
+
+	logger, _ := logging.NewLoggerFromConfig(config, component)
+	return logger
+}