Merge pull request #120 from jpinsonneau/skip_tls

NETOBSERV-309 skip TLS checks & add TenantID

api/v1alpha1/flowcollector_types.go

@@ -283,6 +283,11 @@ type FlowCollectorLoki struct {
 	// and querier are int he same host).
 	QuerierURL string `json:"querierUrl,omitempty"`
 
+	//+kubebuilder:default:="netobserv"
+	// TenantID is the Loki X-Scope-OrgID that identifies the tenant for each request.
+	// it will be ignored if instanceSpec is specified
+	TenantID string `json:"tenantID,omitempty"`
+
 	//+kubebuilder:default:="1s"
 	// BatchWait is max time to wait before sending a batch
 	BatchWait metav1.Duration `json:"batchWait,omitempty"`

config/crd/bases/flows.netobserv.io_flowcollectors.yaml

@@ -1529,6 +1529,12 @@ spec:
                     description: StaticLabels is a map of common labels to set on
                       each flow
                     type: object
+                  tenantID:
+                    default: netobserv
+                    description: TenantID is the Loki X-Scope-OrgID that identifies
+                      the tenant for each request. it will be ignored if instanceSpec
+                      is specified
+                    type: string
                   timeout:
                     default: 10s
                     description: Timeout is the maximum time connection / request

controllers/consoleplugin/consoleplugin_objects.go

@@ -103,6 +103,9 @@ func buildArgs(desired *flowsv1alpha1.FlowCollectorConsolePlugin, desiredLoki *f
 		"-key", "/var/serving-cert/tls.key",
 		"-loki", querierURL(desiredLoki),
 		"-loki-labels", strings.Join(constants.LokiIndexFields, ","),
+		"-loki-tenant-id", desiredLoki.TenantID,
+		//TODO: add loki tls config https://issues.redhat.com/browse/NETOBSERV-309
+		"-loki-skip-tls", "true",
 		"-loglevel", desired.LogLevel,
 		"-frontend-config", configPath + configFile,
 	}
@@ -137,6 +140,9 @@ func (b *builder) podTemplate(cmDigest string) *corev1.PodTemplateSpec {
 					"-key", "/var/serving-cert/tls.key",
 					"-loki", querierURL(b.desiredLoki),
 					"-loki-labels", strings.Join(constants.LokiIndexFields, ","),
+					"-loki-tenant-id", b.desiredLoki.TenantID,
+					//TODO: add loki tls config https://issues.redhat.com/browse/NETOBSERV-309
+					"-loki-skip-tls", "true",
 					"-loglevel", b.desired.LogLevel,
 					"-frontend-config", configPath + configFile,
 				},

controllers/consoleplugin/consoleplugin_test.go

@@ -1,6 +1,7 @@
 package consoleplugin
 
 import (
+	"encoding/json"
 	"fmt"
 	"testing"
 
@@ -12,6 +13,8 @@ import (
 
 	flowsv1alpha1 "github.com/netobserv/network-observability-operator/api/v1alpha1"
 	"github.com/netobserv/network-observability-operator/controllers/constants"
+
+	promConfig "github.com/prometheus/common/config"
 )
 
 const testImage = "quay.io/netobserv/network-observability-console-plugin:dev"
@@ -22,6 +25,8 @@ var testArgs = []string{
 	"-key", "/var/serving-cert/tls.key",
 	"-loki", "http://loki:3100/",
 	"-loki-labels", "SrcK8S_Namespace,SrcK8S_OwnerName,DstK8S_Namespace,DstK8S_OwnerName,FlowDirection",
+	"-loki-tenant-id", "netobserv",
+	"-loki-skip-tls", "true",
 	"-loglevel", "info",
 	"-frontend-config", "/opt/app-root/config.yaml",
 }
@@ -128,7 +133,7 @@ func TestContainerUpdateCheck(t *testing.T) {
 
 	//equals specs
 	podSpec, containerConfig := getContainerSpecs()
-	loki := &flowsv1alpha1.FlowCollectorLoki{URL: "http://loki:3100/"}
+	loki := &flowsv1alpha1.FlowCollectorLoki{URL: "http://loki:3100/", TenantID: "netobserv"}
 	fmt.Printf("%v\n", buildArgs(&containerConfig, loki))
 	assert.Equal(containerNeedsUpdate(&podSpec, &containerConfig, loki), false)
 
@@ -181,7 +186,7 @@ func TestBuiltContainer(t *testing.T) {
 
 	//newly created containers should not need update
 	plugin := getPluginConfig()
-	loki := &flowsv1alpha1.FlowCollectorLoki{URL: "http://foo:1234"}
+	loki := &flowsv1alpha1.FlowCollectorLoki{URL: "http://foo:1234", TenantID: "netobserv"}
 	builder := newBuilder(testNamespace, &plugin, loki)
 	newContainer := builder.podTemplate("digest")
 	assert.Equal(containerNeedsUpdate(&newContainer.Spec, &plugin, loki), false)
@@ -246,3 +251,29 @@ func TestAutoScalerUpdateCheck(t *testing.T) {
 	autoScalerSpec.Namespace = "NewNamespace"
 	assert.Equal(autoScalerNeedsUpdate(&autoScalerSpec, &plugin, testNamespace), true)
 }
+
+//ensure HTTPClientConfig Marshal / Unmarshal works as expected for ProxyURL *URL
+//ProxyURL should not be set when only TLSConfig.InsecureSkipVerify is specified
+func TestHTTPClientConfig(t *testing.T) {
+	config := promConfig.HTTPClientConfig{
+		TLSConfig: promConfig.TLSConfig{
+			InsecureSkipVerify: true,
+		},
+	}
+	err := config.Validate()
+	assert.Nil(t, err)
+
+	bs, _ := json.Marshal(config)
+	assert.Equal(t, string(bs), `{"proxy_url":null,"tls_config":{"insecure_skip_verify":true},"follow_redirects":false}`)
+
+	config2 := promConfig.HTTPClientConfig{}
+	err = json.Unmarshal(bs, &config2)
+	assert.Nil(t, err)
+	assert.Equal(t, config2.TLSConfig.InsecureSkipVerify, true)
+	assert.Equal(t, config2.ProxyURL, promConfig.URL{})
+
+	err = config2.Validate()
+	assert.Nil(t, err)
+	assert.Equal(t, config2.TLSConfig.InsecureSkipVerify, true)
+	assert.Nil(t, config2.ProxyURL.URL, nil)
+}

controllers/flowlogspipeline/flp_objects.go

@@ -16,6 +16,8 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 
+	promConfig "github.com/prometheus/common/config"
+
 	flowsv1alpha1 "github.com/netobserv/network-observability-operator/api/v1alpha1"
 	"github.com/netobserv/network-observability-operator/controllers/constants"
 	"github.com/netobserv/network-observability-operator/pkg/helper"
@@ -243,6 +245,13 @@ func (b *builder) addTransformStages(lastStage *config.PipelineBuilderStage) {
 		lokiWrite.URL = b.desiredLoki.URL
 		lokiWrite.TimestampLabel = "TimeFlowEndMs"
 		lokiWrite.TimestampScale = "1ms"
+		lokiWrite.TenantID = b.desiredLoki.TenantID
+		//TODO: set proper tls config https://issues.redhat.com/browse/NETOBSERV-309
+		lokiWrite.ClientConfig = &promConfig.HTTPClientConfig{
+			TLSConfig: promConfig.TLSConfig{
+				InsecureSkipVerify: true,
+			},
+		}
 	}
 	enrichedStage.WriteLoki("loki", lokiWrite)
 

docs/FlowCollector.md

@@ -2682,6 +2682,15 @@ Settings related to the Loki client, used as a flow store.
             <i>Default</i>: map[app:netobserv-flowcollector]<br/>
         </td>
         <td>false</td>
+      </tr><tr>
+        <td><b>tenantID</b></td>
+        <td>string</td>
+        <td>
+          TenantID is the Loki X-Scope-OrgID that identifies the tenant for each request. it will be ignored if instanceSpec is specified<br/>
+          <br/>
+            <i>Default</i>: netobserv<br/>
+        </td>
+        <td>false</td>
       </tr><tr>
         <td><b>timeout</b></td>
         <td>string</td>

go.mod

@@ -18,3 +18,5 @@ require (
 	k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9
 	sigs.k8s.io/controller-runtime v0.11.0
 )
+
+replace github.com/prometheus/common v0.32.1 => github.com/netobserv/prometheus-common v0.31.2-0.20220720134304-43e74fd22881

go.sum

@@ -775,6 +775,8 @@ github.com/netobserv/flowlogs-pipeline v0.1.2-0.20220616154151-f71171409f0b/go.m
 github.com/netobserv/gopipes v0.1.1/go.mod h1:eGoHZW1ON8Dx/zmDXUhsbVNqatPjtpdO0UZBmGZGmVI=
 github.com/netobserv/loki-client-go v0.0.0-20211018150932-cb17208397a9/go.mod h1:LHXpc5tjKvsfZn0pwLKrvlgEhZcCaw3Di9mUEZGAI4E=
 github.com/netobserv/netobserv-ebpf-agent v0.1.1-0.20220608092850-3fd4695b7cc2/go.mod h1:996FEHp8Xj+AKCkiN4eH3dl/yF2DzuYM0kchWZOrapM=
+github.com/netobserv/prometheus-common v0.31.2-0.20220720134304-43e74fd22881 h1:hx5bi6xBovRjmwUoVJBzhJ3EDo4K4ZUsqqKrJuQ2vMI=
+github.com/netobserv/prometheus-common v0.31.2-0.20220720134304-43e74fd22881/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
 github.com/netsampler/goflow2 v1.1.1-0.20220509155230-5300494e4785/go.mod h1:yqw2cLe+lbnDN1+JKBqxoj2FKOA83iB8wV0aCKnlesg=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
@@ -894,8 +896,6 @@ github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9
 github.com/prometheus/common v0.28.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
 github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
 github.com/prometheus/common v0.31.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
-github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4=
-github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=

vendor/modules.txt

@@ -139,7 +139,7 @@ github.com/prometheus/client_golang/prometheus/internal
 github.com/prometheus/client_golang/prometheus/promhttp
 # github.com/prometheus/client_model v0.2.0
 github.com/prometheus/client_model/go
-# github.com/prometheus/common v0.32.1
+# github.com/prometheus/common v0.32.1 => github.com/netobserv/prometheus-common v0.31.2-0.20220720134304-43e74fd22881
 ## explicit
 github.com/prometheus/common/config
 github.com/prometheus/common/expfmt