Add plugin manager for sources

.goreleaser.plugin.yaml

@@ -17,3 +17,17 @@ builds:
       - arm64
     goarm:
       - 7
+  - id: cm-watcher
+    binary: source_cm-watcher_{{ .Os }}_{{ .Arch }}
+    no_unique_dist_dir: true
+    main: cmd/source/cm-watcher/main.go
+    env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+      - darwin
+    goarch:
+      - amd64
+      - arm64
+    goarm:
+      - 7

CONTRIBUTING.md

@@ -36,7 +36,7 @@ This section describes how to build and run Botkube from source code.
      This is ideal for running Botkube on a local cluster, e.g. using [kind](https://kind.sigs.k8s.io) or [`minikube`](https://minikube.sigs.k8s.io/docs/).
 
      Remember to set the `IMAGE_PLATFORM` env var to your target architecture. By default, the build targets `linux/amd64`.
-     
+
      For example, the command below builds the `linux/arm64` target:
 
         ```sh
@@ -62,23 +62,23 @@ This section describes how to build and run Botkube from source code.
      docker tag ghcr.io/kubeshop/botkube:v9.99.9-dev-amd64 {your_account}/botkube:v9.99.9-dev
      docker push {your_account}/botkube:v9.99.9-dev
      ```
-     
+
      Where `{your_account}` is Docker hub or any other registry provider account to which you can push the image.
 
 2. Install Botkube with any of communication platform configured, according to [the installation instructions](https://docs.botkube.io/installation/). During the Helm chart installation step, set the following flags:
-   
+
    ```sh
    export IMAGE_REGISTRY="{imageRegistry}" # e.g. docker.io
    export IMAGE_PULL_POLICY="{pullPolicy}" # e.g. Always or IfNotPresent
-   
+
    --set image.registry=${IMAGE_REGISTRY} \
    --set image.repository={your_account}/botkube \
    --set image.tag=v9.99.9-dev \
    --set image.pullPolicy=${IMAGE_PULL_POLICY}
    ```
-   
+
    Check [values.yaml](./helm/botkube/values.yaml) for default options.
-   
+
 ### Build and run locally
 
 For faster development, you can also build and run Botkube outside K8s cluster.
@@ -130,6 +130,44 @@ For faster development, you can also build and run Botkube outside K8s cluster.
    ./botkube
    ```
 
+#### Develop Botkube plugins
+
+**Prerequisite**
+
+- Being able to start the Botkube binary locally.
+- [GoReleaser](https://goreleaser.com/install/)
+
+**Steps**
+
+1. Start fake plugins server to serve binaries from [`dist`](dist) folder:
+
+   ```bash
+   go run test/helpers/plugin_server.go
+   ```
+
+	 > **Note**
+	 > If Botkube runs inside the k3d cluster, export the `PLUGIN_SERVER_HOST=http://host.k3d.internal` environment variable.
+
+2. Export Botkube plugins cache directory:
+
+   ```bash
+   export BOTKUBE_PLUGINS_CACHE__DIR="/tmp/plugins"
+   ```
+
+3. In other terminal window, run:
+
+   ```bash
+   # rebuild plugins only for current GOOS and GOARCH
+   make build-plugins-single &&
+   # remove cached plugins
+   rm -rf $BOTKUBE_PLUGINS_CACHE__DIR &&
+   # start botkube to download fresh plugins
+   ./botkube
+   ```
+
+   > **Note**
+   > Each time you make a change to the [source](cmd/source) or [executors](cmd/executor) plugins re-run the above command.
+
 ## Making A Change
 
 - Before making any significant changes, please [open an issue](https://github.com/kubeshop/botkube/issues). Discussing your proposed changes ahead of time will make the contribution process smooth for everyone.

Makefile

@@ -1,5 +1,5 @@
 .DEFAULT_GOAL := build
-.PHONY: container-image test test-integration-slack test-integration-discord build pre-build publish lint lint-fix go-import-fmt system-check save-images load-and-push-images gen-grpc-resources build-plugins
+.PHONY: container-image test test-integration-slack test-integration-discord build pre-build publish lint lint-fix go-import-fmt system-check save-images load-and-push-images gen-grpc-resources build-plugins build-plugins-single
 
 # Show this help.
 help:
@@ -18,21 +18,28 @@ test: system-check
 	@go test -v  -race ./...
 
 test-integration-slack: system-check
-	@go test -v -tags=integration -race -count=1 ./test/... -run "TestSlack"
+	@go test -v -tags=integration -race -count=1 ./test/e2e/... -run "TestSlack"
 
 test-integration-discord: system-check
-	@go test -v -tags=integration -race -count=1 ./test/... -run "TestDiscord"
+	@go test -v -tags=integration -race -count=1 ./test/e2e/... -run "TestDiscord"
 
 # Build the binary
 build: pre-build
 	@cd cmd/botkube;GOOS_VAL=$(shell go env GOOS) CGO_ENABLED=0 GOARCH_VAL=$(shell go env GOARCH) go build -o $(shell go env GOPATH)/bin/botkube
 	@echo "Build completed successfully"
 
+# Build Botkube official plugins for all supported platforms.
 build-plugins: pre-build
 	@echo "Building plugins binaries"
 	@./hack/goreleaser.sh build_plugins
 	@echo "Build completed successfully"
 
+# Build Botkube official plugins only for current GOOS and GOARCH.
+build-plugins-single: pre-build
+	@echo "Building single target plugins binaries"
+	@./hack/goreleaser.sh build_plugins_single
+	@echo "Build completed successfully"
+
 # Build the image
 container-image: pre-build
 	@echo "Building docker image"

cmd/botkube/main.go

@@ -30,6 +30,7 @@ import (
 	"github.com/kubeshop/botkube/internal/analytics"
 	"github.com/kubeshop/botkube/internal/lifecycle"
 	"github.com/kubeshop/botkube/internal/plugin"
+	"github.com/kubeshop/botkube/internal/source"
 	"github.com/kubeshop/botkube/internal/storage"
 	"github.com/kubeshop/botkube/pkg/action"
 	"github.com/kubeshop/botkube/pkg/bot"
@@ -43,7 +44,6 @@ import (
 	"github.com/kubeshop/botkube/pkg/notifier"
 	"github.com/kubeshop/botkube/pkg/recommendation"
 	"github.com/kubeshop/botkube/pkg/sink"
-	"github.com/kubeshop/botkube/pkg/source"
 )
 
 const (
@@ -100,8 +100,8 @@ func run() error {
 	errGroup, ctx := errgroup.WithContext(ctx)
 
 	collector := plugin.NewCollector(logger)
-	enabledPluginExecutors := collector.GetAllEnabledAndUsedPlugins(conf)
-	pluginManager := plugin.NewManager(logger, conf.Plugins, enabledPluginExecutors)
+	enabledPluginExecutors, enabledPluginSources := collector.GetAllEnabledAndUsedPlugins(conf)
+	pluginManager := plugin.NewManager(logger, conf.Plugins, enabledPluginExecutors, enabledPluginSources)
 
 	err = pluginManager.Start(ctx)
 	if err != nil {
@@ -322,6 +322,12 @@ func run() error {
 	actionProvider := action.NewProvider(logger.WithField(componentLogFieldKey, "Action Provider"), conf.Actions, executorFactory)
 	router.AddEnabledActionBindings(conf.Actions)
 
+	sourcePluginDispatcher := source.NewDispatcher(logger, notifiers, pluginManager, conf)
+	err = sourcePluginDispatcher.Start(ctx)
+	if err != nil {
+		return fmt.Errorf("while starting source plugin event dispatcher: %w", err)
+	}
+
 	// Create and start controller
 	ctrl := controller.New(
 		logger.WithField(componentLogFieldKey, "Controller"),

cmd/executor/echo/README.md

@@ -4,7 +4,7 @@ Echo is the example Botkube executor used during [e2e tests](../../../test/e2e).
 
 ## Configuration parameters
 
-The Echo configuration should be specified in YAML format. It accepts such parameters:
+The configuration should be specified in YAML format. Such parameters are supported:
 
 ```yaml
 changeResponseToUpperCase: true # default is 'false'.

cmd/source/cm-watcher/README.md

@@ -0,0 +1,13 @@
+# ConfigMap watcher source
+
+ConfigMap watcher source is the example Botkube source used during [e2e tests](../../../test/e2e).
+
+## Configuration parameters
+
+The configuration should be specified in YAML format. Such parameters are supported:
+
+```yaml
+configMap:
+  name: cm-map-watcher # config map name to react to
+  namespace: botkube  # config map namespace
+```

cmd/source/cm-watcher/main.go

@@ -0,0 +1,106 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/hashicorp/go-plugin"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/tools/clientcmd"
+	watchtools "k8s.io/client-go/tools/watch"
+
+	"github.com/kubeshop/botkube/pkg/api/source"
+)
+
+const pluginName = "cm-watcher"
+
+type (
+	// Config holds executor configuration.
+	Config struct {
+		ConfigMap Object
+	}
+	// Object holds information about object to watch.
+	Object struct {
+		Name      string          `yaml:"name"`
+		Namespace string          `yaml:"namespace"`
+		Event     watch.EventType `yaml:"event"`
+	}
+)
+
+// CMWatcher implements Botkube source plugin.
+type CMWatcher struct{}
+
+// Stream returns a given command as response.
+func (CMWatcher) Stream(ctx context.Context) (source.StreamOutput, error) {
+	// TODO: in request we should receive the source configurations.
+	cfg := Config{
+		ConfigMap: Object{
+			Name:      "cm-watcher-trigger",
+			Namespace: "botkube",
+			Event:     "ADDED",
+		},
+	}
+
+	out := source.StreamOutput{
+		Output: make(chan []byte),
+	}
+
+	go listenEvents(ctx, cfg.ConfigMap, out.Output)
+
+	return out, nil
+}
+
+func listenEvents(ctx context.Context, obj Object, sink chan<- []byte) {
+	config, err := clientcmd.BuildConfigFromFlags("", os.Getenv("KUBECONFIG"))
+	exitOnError(err)
+	clientset, err := kubernetes.NewForConfig(config)
+	exitOnError(err)
+
+	fieldSelector := fields.OneTermEqualSelector("metadata.name", obj.Name).String()
+	lw := &cache.ListWatch{
+		ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
+			options.FieldSelector = fieldSelector
+			return clientset.CoreV1().ConfigMaps(obj.Namespace).List(ctx, options)
+		},
+		WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
+			options.FieldSelector = fieldSelector
+			return clientset.CoreV1().ConfigMaps(obj.Namespace).Watch(ctx, options)
+		},
+	}
+
+	infiniteWatch := func(event watch.Event) (bool, error) {
+		if event.Type == obj.Event {
+			cm := event.Object.(*corev1.ConfigMap)
+			msg := fmt.Sprintf("Detected `%s` event on `%s/%s`", obj.Event, cm.Namespace, cm.Name)
+			sink <- []byte(msg)
+		}
+
+		// always continue - context will cancel this watch for us :)
+		return false, nil
+	}
+
+	_, err = watchtools.UntilWithSync(ctx, lw, &corev1.ConfigMap{}, nil, infiniteWatch)
+	exitOnError(err)
+}
+
+func main() {
+	source.Serve(map[string]plugin.Plugin{
+		pluginName: &source.Plugin{
+			Source: &CMWatcher{},
+		},
+	})
+}
+
+func exitOnError(err error) {
+	if err != nil {
+		log.Fatal(err)
+	}
+}

hack/goreleaser.sh

@@ -96,6 +96,10 @@ build_plugins() {
   goreleaser build -f .goreleaser.plugin.yaml --rm-dist --snapshot
 }
 
+build_plugins_single() {
+  goreleaser build -f .goreleaser.plugin.yaml --rm-dist --snapshot --single-target
+}
+
 build_single() {
   export IMAGE_TAG=v9.99.9-dev
   docker run --rm --privileged \
@@ -127,6 +131,9 @@ case "${1}" in
   build_plugins)
     build_plugins
     ;;
+  build_plugins_single)
+    build_plugins_single
+    ;;
   build_single)
     build_single
     ;;

helm/botkube/e2e-test-values.yaml

@@ -20,6 +20,7 @@ communications:
               - k8s-events
               - k8s-annotated-cm-delete
               - k8s-pod-create-events
+              - plugin-based
         'secondary':
           name: "" # Tests will override this temporarily
           notification:
@@ -47,6 +48,7 @@ communications:
               - k8s-events
               - k8s-annotated-cm-delete
               - k8s-pod-create-events
+              - plugin-based
         'secondary':
           id: "" # Tests will override this channel ID temporarily
           notification:
@@ -120,6 +122,13 @@ sources:
           event: # overrides top level `event` entry
             types:
               - update
+
+  'plugin-based':
+    botkube/cm-watcher:
+      enabled: true
+      config:
+        configMapName: cm-watcher-trigger
+
 executors:
   'kubectl-read-only':
     kubectl: