NETOBSERV-226 Refactor query API and filters management

cmd/plugin-backend.go

@@ -10,7 +10,7 @@ import (
 
 	"github.com/sirupsen/logrus"
 
-	"github.com/netobserv/network-observability-console-plugin/pkg/handler"
+	"github.com/netobserv/network-observability-console-plugin/pkg/loki"
 	"github.com/netobserv/network-observability-console-plugin/pkg/server"
 )
 
@@ -72,12 +72,7 @@ func main() {
 		CORSAllowMethods: *corsMethods,
 		CORSAllowHeaders: *corsHeaders,
 		CORSMaxAge:       *corsMaxAge,
-		Loki: handler.LokiConfig{
-			URL:      lURL,
-			Timeout:  *lokiTimeout,
-			TenantID: *lokiTenantID,
-			Labels:   strings.Split(lLabels, ","),
-		},
-		FrontendConfig: *frontendConfig,
+		Loki:             loki.NewConfig(lURL, *lokiTimeout, *lokiTenantID, strings.Split(lLabels, ",")),
+		FrontendConfig:   *frontendConfig,
 	})
 }

pkg/handler/csv/loki_csv.go

@@ -13,67 +13,58 @@ const (
 )
 
 func GetCSVData(qr *model.QueryResponse, columns []string) ([][]string, error) {
-	if columns != nil && len(columns) == 0 {
-		return nil, fmt.Errorf("columns can't be empty if specified")
-	}
-
-	if streams, ok := qr.Data.Result.(model.Streams); ok {
-		return manageStreams(streams, columns)
-	}
-	return nil, fmt.Errorf("loki returned an unexpected type: %T", qr.Data.Result)
-}
-
-func manageStreams(streams model.Streams, columns []string) ([][]string, error) {
-	//make csv datas containing header as first line + rows
-	datas := make([][]string, 1)
-	//prepare columns for faster lookup
-	columnsMap := utils.GetMapInterface(columns)
-	//set Timestamp as first data
-	includeTimestamp := false
-	if _, exists := columnsMap[timestampCol]; exists || len(columns) == 0 {
-		datas[0] = append(datas[0], timestampCol)
-		includeTimestamp = true
-	}
-	//keep ordered labels / field names between each lines
-	//filtered by columns parameter if specified
-	var labels []string
-	var fields []string
-	for _, stream := range streams {
-		//get labels from first stream
-		if labels == nil {
-			labels = make([]string, 0, len(stream.Labels))
-			for name := range stream.Labels {
-				if _, exists := columnsMap[name]; exists || len(columns) == 0 {
-					labels = append(fields, name)
+	if streams, ok := qr.Data.Result.(model.Streams); ok { //make csv datas containing header as first line + rows
+		data := make([][]string, 1)
+		//prepare columns for faster lookup
+		columnsMap := utils.GetMapInterface(columns)
+		//set Timestamp as first data
+		includeTimestamp := false
+		if _, exists := columnsMap[timestampCol]; exists || len(columns) == 0 {
+			data[0] = append(data[0], timestampCol)
+			includeTimestamp = true
+		}
+		//keep ordered labels / field names between each lines
+		//filtered by columns parameter if specified
+		var labels []string
+		var fields []string
+		for _, stream := range streams {
+			//get labels from first stream
+			if labels == nil {
+				labels = make([]string, 0, len(stream.Labels))
+				for name := range stream.Labels {
+					if _, exists := columnsMap[name]; exists || len(columns) == 0 {
+						labels = append(fields, name)
+					}
 				}
+				data[0] = append(data[0], labels...)
 			}
-			datas[0] = append(datas[0], labels...)
-		}
 
-		//apply timestamp & labels for each entries and add json line fields
-		for _, entry := range stream.Entries {
-			//get json line
-			var line map[string]interface{}
-			err := json.Unmarshal([]byte(entry.Line), &line)
-			if err != nil {
-				return nil, fmt.Errorf("cannot unmarshal line %s", entry.Line)
-			}
+			//apply timestamp & labels for each entries and add json line fields
+			for _, entry := range stream.Entries {
+				//get json line
+				var line map[string]interface{}
+				err := json.Unmarshal([]byte(entry.Line), &line)
+				if err != nil {
+					return nil, fmt.Errorf("cannot unmarshal line %s", entry.Line)
+				}
 
-			//get fields from first line
-			if fields == nil {
-				fields = make([]string, 0, len(line))
-				for name := range line {
-					if _, exists := columnsMap[name]; exists || len(columns) == 0 {
-						fields = append(fields, name)
+				//get fields from first line
+				if fields == nil {
+					fields = make([]string, 0, len(line))
+					for name := range line {
+						if _, exists := columnsMap[name]; exists || len(columns) == 0 {
+							fields = append(fields, name)
+						}
 					}
+					data[0] = append(data[0], fields...)
 				}
-				datas[0] = append(datas[0], fields...)
-			}
 
-			datas = append(datas, getRowDatas(stream, entry, labels, fields, line, len(datas[0]), includeTimestamp))
+				data = append(data, getRowDatas(stream, entry, labels, fields, line, len(data[0]), includeTimestamp))
+			}
 		}
+		return data, nil
 	}
-	return datas, nil
+	return nil, fmt.Errorf("loki returned an unexpected type: %T", qr.Data.Result)
 }
 
 func getRowDatas(stream model.Stream, entry model.Entry, labels, fields []string,

pkg/handler/export.go

@@ -0,0 +1,43 @@
+package handler
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+
+	"github.com/netobserv/network-observability-console-plugin/pkg/loki"
+)
+
+const (
+	exportCSVFormat  = "csv"
+	exportFormatKey  = "format"
+	exportcolumnsKey = "columns"
+)
+
+func ExportFlows(cfg loki.Config) func(w http.ResponseWriter, r *http.Request) {
+	lokiClient := newLokiClient(&cfg)
+
+	return func(w http.ResponseWriter, r *http.Request) {
+		params := r.URL.Query()
+		hlog.Debugf("ExportFlows query params: %s", params)
+
+		flows, code, err := getFlows(cfg, lokiClient, params)
+		if err != nil {
+			writeError(w, code, err.Error())
+			return
+		}
+
+		exportFormat := params.Get(exportFormatKey)
+		var exportColumns []string
+		if str := params.Get(exportcolumnsKey); len(str) > 0 {
+			exportColumns = strings.Split(str, ",")
+		}
+
+		switch exportFormat {
+		case exportCSVFormat:
+			writeCSV(w, http.StatusOK, flows, exportColumns)
+		default:
+			writeError(w, http.StatusBadRequest, fmt.Sprintf("export format %q is not valid", exportFormat))
+		}
+	}
+}

pkg/handler/flows.go

@@ -0,0 +1,136 @@
+package handler
+
+import (
+	"errors"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/netobserv/network-observability-console-plugin/pkg/httpclient"
+	"github.com/netobserv/network-observability-console-plugin/pkg/loki"
+)
+
+const (
+	startTimeKey = "startTime"
+	endTimeKey   = "endTime"
+	timeRangeKey = "timeRange"
+	limitKey     = "limit"
+	reporterKey  = "reporter"
+	filtersKey   = "filters"
+)
+
+type errorWithCode struct {
+	err  error
+	code int
+}
+
+type keyValues = []string
+
+// Example of raw filters (url-encoded):
+// foo=a,b&bar=c|baz=d
+func parseFilters(raw string) ([][]keyValues, error) {
+	var parsed [][]keyValues
+	decoded, err := url.QueryUnescape(raw)
+	if err != nil {
+		return nil, err
+	}
+	groups := strings.Split(decoded, "|")
+	for _, group := range groups {
+		var andFilters []keyValues
+		filters := strings.Split(group, "&")
+		for _, filter := range filters {
+			pair := strings.Split(filter, "=")
+			if len(pair) == 2 {
+				andFilters = append(andFilters, pair)
+			}
+		}
+		parsed = append(parsed, andFilters)
+	}
+	return parsed, nil
+}
+
+func getStartTime(params url.Values) (string, error) {
+	start := params.Get(startTimeKey)
+	if len(start) == 0 {
+		tr := params.Get(timeRangeKey)
+		if len(tr) > 0 {
+			r, err := strconv.ParseInt(tr, 10, 64)
+			if err != nil {
+				return "", errors.New("Could not parse time range: " + err.Error())
+			}
+			start = strconv.FormatInt(time.Now().Unix()-r, 10)
+		}
+	}
+	return start, nil
+}
+
+func GetFlows(cfg loki.Config) func(w http.ResponseWriter, r *http.Request) {
+	lokiClient := newLokiClient(&cfg)
+
+	return func(w http.ResponseWriter, r *http.Request) {
+		params := r.URL.Query()
+		hlog.Debugf("GetFlows query params: %s", params)
+
+		flows, code, err := getFlows(cfg, lokiClient, params)
+		if err != nil {
+			writeError(w, code, err.Error())
+			return
+		}
+
+		writeRawJSON(w, http.StatusOK, flows)
+	}
+}
+
+func getFlows(cfg loki.Config, client httpclient.HTTPClient, params url.Values) ([]byte, int, error) {
+	start, err := getStartTime(params)
+	if err != nil {
+		return nil, http.StatusBadRequest, err
+	}
+	end := params.Get(endTimeKey)
+	limit := params.Get(limitKey)
+	reporter := params.Get(reporterKey)
+	rawFilters := params.Get(filtersKey)
+	filterGroups, err := parseFilters(rawFilters)
+	if err != nil {
+		return nil, http.StatusBadRequest, err
+	}
+
+	var rawJSON []byte
+	if len(filterGroups) > 1 {
+		// match any, and multiple filters => run in parallel then aggregate
+		var queries []string
+		for _, group := range filterGroups {
+			qb := loki.NewFlowQueryBuilder(&cfg, start, end, limit, reporter)
+			err := qb.Filters(group)
+			if err != nil {
+				return nil, http.StatusBadRequest, errors.New("Can't build query: " + err.Error())
+			}
+			queries = append(queries, qb.Build())
+		}
+		res, code, err := fetchParallel(client, queries)
+		if err != nil {
+			return nil, code, errors.New("Error while fetching flows from Loki: " + err.Error())
+		}
+		rawJSON = res
+	} else {
+		// else, run all at once
+		qb := loki.NewFlowQueryBuilder(&cfg, start, end, limit, reporter)
+		if len(filterGroups) > 0 {
+			err := qb.Filters(filterGroups[0])
+			if err != nil {
+				return nil, http.StatusBadRequest, err
+			}
+		}
+		query := qb.Build()
+		resp, code, err := executeLokiQuery(query, client)
+		if err != nil {
+			return nil, code, errors.New("Error while fetching flows from Loki: " + err.Error())
+		}
+		rawJSON = resp
+	}
+
+	hlog.Tracef("GetFlows raw response: %v", string(rawJSON))
+	return rawJSON, http.StatusOK, nil
+}

pkg/handler/flows_test.go

@@ -0,0 +1,63 @@
+package handler
+
+import (
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestParseFilters(t *testing.T) {
+	// 2 groups
+	groups, err := parseFilters(url.QueryEscape("foo=a,b&bar=c|baz=d"))
+	require.NoError(t, err)
+
+	assert.Len(t, groups, 2)
+	assert.Equal(t, [][]string{
+		{"foo", "a,b"},
+		{"bar", "c"},
+	}, groups[0])
+	assert.Equal(t, [][]string{
+		{"baz", "d"},
+	}, groups[1])
+
+	// Resource path + port, match all
+	groups, err = parseFilters(url.QueryEscape(`SrcK8S_Type="Pod"&SrcK8S_Namespace="default"&SrcK8S_Name="test"&SrcPort=8080`))
+	require.NoError(t, err)
+
+	assert.Len(t, groups, 1)
+	assert.Equal(t, [][]string{
+		{"SrcK8S_Type", `"Pod"`},
+		{"SrcK8S_Namespace", `"default"`},
+		{"SrcK8S_Name", `"test"`},
+		{"SrcPort", `8080`},
+	}, groups[0])
+
+	// Resource path + port, match any
+	groups, err = parseFilters(url.QueryEscape(`SrcK8S_Type="Pod"&SrcK8S_Namespace="default"&SrcK8S_Name="test"|SrcPort=8080`))
+	require.NoError(t, err)
+
+	assert.Len(t, groups, 2)
+	assert.Equal(t, [][]string{
+		{"SrcK8S_Type", `"Pod"`},
+		{"SrcK8S_Namespace", `"default"`},
+		{"SrcK8S_Name", `"test"`},
+	}, groups[0])
+
+	assert.Equal(t, [][]string{
+		{"SrcPort", `8080`},
+	}, groups[1])
+
+	// Resource path + name, match all
+	groups, err = parseFilters(url.QueryEscape(`SrcK8S_Type="Pod"&SrcK8S_Namespace="default"&SrcK8S_Name="test"&SrcK8S_Name="nomatch"`))
+	require.NoError(t, err)
+
+	assert.Len(t, groups, 1)
+	assert.Equal(t, [][]string{
+		{"SrcK8S_Type", `"Pod"`},
+		{"SrcK8S_Namespace", `"default"`},
+		{"SrcK8S_Name", `"test"`},
+		{"SrcK8S_Name", `"nomatch"`},
+	}, groups[0])
+}