Merge with v3

.github/actions/update-docs/action.yml

@@ -15,10 +15,10 @@ inputs:
     required: true
   version:
     description: "Version name to be deployed by mike"
-    required: true 
+    required: true
   release:
     description: "Determines if the set version is a stable and latest version, otherwise it is a dev version. (Default false)"
-    default: 'false'
+    default: "false"
     required: false
 
 runs:
@@ -35,14 +35,13 @@ runs:
       run: |
         poetry install --with docs
 
-    - name: Update ${{ github.head_ref }} branch
+    - name: Update gh-pages branch
       shell: bash
       run: |
         git config --local user.name  ${{ inputs.username }}
         git config --local user.email ${{ inputs.email }}
         git config --local user.password ${{ inputs.token }}
-
-        git pull 
+        git fetch origin gh-pages
 
     - name: Deploy ${{ inputs.version }} version of the documentation with mike
       shell: bash

README.md

@@ -29,6 +29,10 @@ You can install it with [pip](https://github.com/pypa/pip):
 pip install kpops
 ```
 
+# GitHub action
+
+Please refer to the [GitHub Actions section](https://bakdata.github.io/kpops/latest/user/references/ci-integration/github-actions) for the documentation.
+
 ## Contributing
 
 We are happy if you want to contribute to this project.

actions/kpops-runner/action.yaml → action.yaml

@@ -11,17 +11,14 @@ inputs:
   working-directory:
     description: "The root directory containing the config.yaml, pipelines folder and defaults"
     default: "."
-  pipeline-base-dir:
-    description: "Directory where relative pipeline variables are initialized from"
-    required: false
   defaults:
     description: "Defaults folder path"
     required: false
   config:
-    description: "config.yaml file path"
+    description: "Directory containing the config*.yaml file(s)"
     required: false
-  components:
-    description: "Components package path"
+  environment:
+    description: "Environment to run KPOps in"
     required: false
   filter-type:
     description: "Whether to include/exclude the steps defined in KPOPS_PIPELINE_STEPS (default is include)"
@@ -73,10 +70,15 @@ runs:
       shell: bash
       run: |
         echo "::group::install kpops package"
-        pip install -r "${{ steps.requirements.outputs.path }}"
+        # Check if kpops-version contains ".dev"
+        if [[ "${{ inputs.kpops-version }}" == *".dev"* ]]; then
+          pip install -r "${{ steps.requirements.outputs.path }}" -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/
+        else
+          pip install -r "${{ steps.requirements.outputs.path }}"
+        fi
         echo "::endgroup::"
 
     - name: ${{ inputs.command }} ${{ inputs.pipeline }} pipeline
       shell: bash
       working-directory: ${{inputs.working-directory}}
-      run: kpops ${{ inputs.command }} ${{ inputs.pipeline }} ${{ inputs.components }} ${{ (inputs.defaults != '' && format('--defaults {0}', inputs.defaults)) || '' }} ${{ (inputs.config != '' && format('--config {0}', inputs.config)) || '' }} ${{ (inputs.pipeline-base-dir != '' && format('--pipeline-base-dir {0}', inputs.pipeline-base-dir)) || '' }} ${{ (inputs.filter-type != '' && format('--filter-type {0}', inputs.filter-type)) || '' }}
+      run: kpops ${{ inputs.command }} ${{ inputs.pipeline }} ${{ (inputs.defaults != '' && format('--defaults {0}', inputs.defaults)) || '' }} ${{ (inputs.config != '' && format('--config {0}', inputs.config)) || '' }} ${{ (inputs.environment != '' && format('--environment {0}', inputs.environment)) || '' }} ${{ (inputs.filter-type != '' && format('--filter-type {0}', inputs.filter-type)) || '' }}

config.yaml

@@ -1,5 +1,3 @@
-environment: development
-brokers: "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092"
-kafka_connect_host: "http://localhost:8083"
-kafka_rest_host: "http://localhost:8082"
-schema_registry_url: "http://localhost:8081"
+kafka_brokers: "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092"
+components_module: tests.pipeline.test_components
+pipeline_base_dir: tests/pipeline

docs/docs/developer/auto-generation.md

@@ -10,7 +10,7 @@ Auto generation happens mostly with [`pre-commit`](https://pre-commit.com/) hook
 
 - `cli_env_vars.env` -- All CLI environment variables in a `dotenv` file.
 - `cli_env_vars.md` -- All CLI environment variables in a table.
-- `config_env_vars.env` -- Almost all pipeline config environment variables in a `dotenv` file. The script checks for each field in [`PipelineConfig`](https://github.com/bakdata/kpops/blob/main/kpops/cli/pipeline_config.py) whether it has an `env` attribute defined. The script is currently unable to visit the classes of fields like `topic_name_config`, hence any environment variables defined there would remain unknown to it.
+- `config_env_vars.env` -- Almost all pipeline config environment variables in a `dotenv` file. The script checks for each field in [`KpopsConfig`](https://github.com/bakdata/kpops/blob/main/kpops/cli/kpops_config.py) whether it has an `env` attribute defined. The script is currently unable to visit the classes of fields like `topic_name_config`, hence any environment variables defined there would remain unknown to it.
 - `config_env_vars.env` -- Almost all pipeline config environment variables in a table.
 - `variable_substitution.yaml` -- A copy of `./tests/pipeline/resources/component-type-substitution/pipeline.yaml` used as an example of substitution.
 

docs/docs/resources/architecture/components-hierarchy.md

@@ -1,20 +1,22 @@
 ```mermaid
 flowchart BT
     KubernetesApp --> PipelineComponent
-    KafkaConnector --> PipelineComponent
-    KafkaApp --> KubernetesApp
+    HelmApp --> KubernetesApp
+    KafkaApp --> HelmApp
     StreamsApp --> KafkaApp
     ProducerApp --> KafkaApp
+    KafkaConnector --> PipelineComponent
     KafkaSourceConnector --> KafkaConnector
     KafkaSinkConnector --> KafkaConnector
-    
-    click KubernetesApp "../kubernetes-app"
-    click KafkaApp "../kafka-app"
-    click StreamsApp "../streams-app"
-    click ProducerApp "../producer-app"
-    click KafkaConnector "../kafka-connector"
-    click KafkaSourceConnector "../kafka-source-connector"
-    click KafkaSinkConnector "../kafka-sink-connector"
+
+    click KubernetesApp "/kpops/user/core-concepts/components/kubernetes-app"
+    click HelmApp "/kpops/user/core-concepts/components/helm-app"
+    click KafkaApp "/kpops/user/core-concepts/components/kafka-app"
+    click StreamsApp "/kpops/user/core-concepts/components/streams-app"
+    click ProducerApp "/kpops/user/core-concepts/components/producer-app"
+    click KafkaConnector "/kpops/user/core-concepts/components/kafka-connector"
+    click KafkaSourceConnector "/kpops/user/core-concepts/components/kafka-source-connector"
+    click KafkaSinkConnector "/kpops/user/core-concepts/components/kafka-sink-connector"
 ```
 
 <p style="text-align: center;"><i>KPOps component hierarchy</i></p>

docs/docs/resources/pipeline-components/dependencies/defaults_pipeline_component_dependencies.yaml

@@ -1,3 +1,6 @@
+helm-app.yaml:
+- app-helm-app.yaml
+- repo_config-helm-app.yaml
 kafka-app.yaml:
 - app-kafka-app.yaml
 - version-kafka-app.yaml
@@ -20,8 +23,6 @@ kubernetes-app.yaml:
 - to.yaml
 - namespace.yaml
 - app-kubernetes-app.yaml
-- repo_config-kubernetes-app.yaml
-- version.yaml
 producer-app.yaml:
 - from_-producer-app.yaml
 - app-producer-app.yaml

docs/docs/resources/pipeline-components/dependencies/kpops_structure.yaml

@@ -1,4 +1,13 @@
 kpops_components_fields:
+  helm-app:
+  - name
+  - prefix
+  - from_
+  - to
+  - namespace
+  - app
+  - repo_config
+  - version
   kafka-app:
   - name
   - prefix
@@ -46,8 +55,6 @@ kpops_components_fields:
   - to
   - namespace
   - app
-  - repo_config
-  - version
   pipeline-component:
   - name
   - prefix
@@ -72,7 +79,8 @@ kpops_components_fields:
   - repo_config
   - version
 kpops_components_inheritance_ref:
-  kafka-app: kubernetes-app
+  helm-app: kubernetes-app
+  kafka-app: helm-app
   kafka-connector: pipeline-component
   kafka-sink-connector: kafka-connector
   kafka-source-connector: kafka-connector

docs/docs/resources/pipeline-components/dependencies/pipeline_component_dependencies.yaml

@@ -1,10 +1,18 @@
+helm-app.yaml:
+- prefix.yaml
+- from_.yaml
+- to.yaml
+- namespace.yaml
+- app-helm-app.yaml
+- repo_config-helm-app.yaml
+- version.yaml
 kafka-app.yaml:
 - prefix.yaml
 - from_.yaml
 - to.yaml
 - namespace.yaml
 - app-kafka-app.yaml
-- repo_config-kubernetes-app.yaml
+- repo_config-helm-app.yaml
 - version-kafka-app.yaml
 kafka-connector.yaml:
 - prefix.yaml
@@ -40,21 +48,19 @@ kubernetes-app.yaml:
 - to.yaml
 - namespace.yaml
 - app-kubernetes-app.yaml
-- repo_config-kubernetes-app.yaml
-- version.yaml
 producer-app.yaml:
 - prefix.yaml
 - from_-producer-app.yaml
 - to.yaml
 - namespace.yaml
 - app-producer-app.yaml
-- repo_config-kubernetes-app.yaml
+- repo_config-helm-app.yaml
 - version-kafka-app.yaml
 streams-app.yaml:
 - prefix.yaml
 - from_.yaml
 - to.yaml
 - namespace.yaml
 - app-streams-app.yaml
-- repo_config-kubernetes-app.yaml
+- repo_config-helm-app.yaml
 - version-kafka-app.yaml

docs/docs/resources/pipeline-components/headers/helm-app.yaml

@@ -0,0 +1,3 @@
+# Kubernetes app managed through Helm with an associated Helm chart
+- type: helm-app
+  name: helm-app # required

docs/docs/resources/pipeline-components/helm-app.yaml

@@ -0,0 +1,63 @@
+# Kubernetes app managed through Helm with an associated Helm chart
+- type: helm-app
+  name: helm-app # required
+  # Pipeline prefix that will prefix every component name. If you wish to not
+  # have any prefix you can specify an empty string.
+  prefix: ${pipeline_name}-
+  from: # Must not be null
+    topics: # read from topic
+      ${pipeline_name}-input-topic:
+        type: input # Implied when role is NOT specified
+      ${pipeline_name}-extra-topic:
+        role: topic-role # Implies `type` to be extra
+      ${pipeline_name}-input-pattern-topic:
+        type: pattern # Implied to be an input pattern if `role` is undefined
+      ${pipeline_name}-extra-pattern-topic:
+        type: pattern # Implied to be an extra pattern if `role` is defined
+        role: some-role
+    components: # read from specific component
+      account-producer:
+        type: output # Implied when role is NOT specified
+      other-producer:
+        role: some-role # Implies `type` to be extra
+      component-as-input-pattern:
+        type: pattern # Implied to be an input pattern if `role` is undefined
+      component-as-extra-pattern:
+        type: pattern # Implied to be an extra pattern if `role` is defined
+        role: some-role
+  # Topic(s) into which the component will write output
+  to:
+    topics:
+      ${pipeline_name}-output-topic:
+        type: output # Implied when role is NOT specified
+      ${pipeline_name}-extra-topic:
+        role: topic-role # Implies `type` to be extra; Will throw an error if `type` is defined
+      ${pipeline_name}-error-topic:
+        type: error
+        # Currently KPOps supports Avro and JSON schemas.
+        key_schema: key-schema # must implement SchemaProvider to use
+        value_schema: value-schema
+        partitions_count: 1
+        replication_factor: 1
+        configs: # https://kafka.apache.org/documentation/#topicconfigs
+          cleanup.policy: compact
+    models: # SchemaProvider is initiated with the values given here
+      model: model
+  namespace: namespace # required
+  # `app` contains application-specific settings, hence it does not have a rigid
+  # structure. The fields below are just an example.
+  app: # required
+    image: exampleImage # Example
+    debug: false # Example
+    commandLine: {} # Example
+  # Helm repository configuration (optional)
+  # If not set the helm repo add will not be called. Useful when using local Helm charts
+  repo_config:
+    repository_name: bakdata-streams-bootstrap # required
+    url: https://bakdata.github.io/streams-bootstrap/ # required
+    repo_auth_flags:
+      username: user
+      password: pass
+      ca_file: /home/user/path/to/ca-file
+      insecure_skip_tls_verify: false
+  version: "1.0.0" # Helm chart version

docs/docs/resources/pipeline-components/kafka-app.yaml

@@ -49,7 +49,7 @@
   # add the key-value pairs they need.
   app: # required
     streams: # required
-      brokers: ${brokers} # required
+      brokers: ${kafka_brokers} # required
       schemaRegistryUrl: ${schema_registry_url}
     nameOverride: override-with-this-name # kafka-app-specific
     imageTag: "1.0.0" # Example values that are shared between streams-app and producer-app

docs/docs/resources/pipeline-components/kubernetes-app.yaml

@@ -50,14 +50,3 @@
     image: exampleImage # Example
     debug: false # Example
     commandLine: {} # Example
-  # Helm repository configuration (optional)
-  # If not set the helm repo add will not be called. Useful when using local Helm charts
-  repo_config:
-    repository_name: bakdata-streams-bootstrap # required
-    url: https://bakdata.github.io/streams-bootstrap/ # required
-    repo_auth_flags:
-      username: user
-      password: pass
-      ca_file: /home/user/path/to/ca-file
-      insecure_skip_tls_verify: false
-  version: "1.0.0" # Helm chart version