Cleanup API for consistency, make it more consumable (breaking changes)

Makefile

@@ -98,9 +98,14 @@ clean: ## Clean
 	go clean ./...
 
 # note: to review coverage execute: go tool cover -html=/tmp/coverage.out
+TEST_OPTS := -race -coverpkg=./... -covermode=atomic -coverprofile=/tmp/coverage.out
 .PHONY: tests-unit
 tests-unit: validate_go ## Unit tests
-	go test -p 1 -race -coverpkg=./... -covermode=atomic -coverprofile=/tmp/coverage.out $$(go list ./... | grep -v /e2e)
+	go test -p 1 $(TEST_OPTS) $$(go list ./... | grep -v /e2e)
+
+.PHONY: tests-fast
+tests-fast: TEST_OPTS=
+tests-fast: tests-unit ## Fast unit tests (no race tests / coverage)
 
 .PHONY: tests-e2e
 tests-e2e: validate_go $(KIND)  ## End-to-end tests

README.md

@@ -433,7 +433,7 @@ service name of `dstPort` port and `protocol` protocol. Unrecognized ports are i
 > Note: `protocol` can be either network protocol name or number  
 >   
 > Note: optionally supports custom network services resolution by defining configuration parameters 
-> `servicesfile` and `protocolsfile` with paths to custom services/protocols files respectively  
+> `servicesFile` and `protocolsFile` with paths to custom services/protocols files respectively  
 
 The fourth rule `add_location` generates new fields with the geo-location information retrieved 
 from DB [ip2location](https://lite.ip2location.com/) based on `dstIP` IP. 
@@ -501,12 +501,12 @@ parameters:
     extract:
       type: aggregates
       aggregates:
-        - Name: "Average key=value for (srcIP, dstIP) pairs"
-          By:
+        - name: "Average key=value for (srcIP, dstIP) pairs"
+          by:
             - "dstIP"
             - "srcIP"
-          Operation: "avg"
-          RecordKey: "value"
+          operation: "avg"
+          recordKey: "value"
 ```
 
 The output fields of the aggregates stage are:
@@ -526,10 +526,10 @@ The pipeline processes flowlogs in batches.
 The output fields with `recent_` prefix are related to the recent batch.
 They are needed when exposing metrics in Prometheus using Counters and Histograms.
 Prometheus Counters API accepts the delta amount to be added to the counter and not the total value as in Gauges.
-In this case, `recent_op_value` and `recent_count` should be used as the `valuekey`.
+In this case, `recent_op_value` and `recent_count` should be used as the `valueKey`.
 The API of Histograms accepts the sample value, so it could be added to the appropriate bucket.
 In this case, we are interested in the raw values of the records in the aggregation group.
-No aggregate operation is needed and it should be set `raw_values`. The `valuekey` should be set to `recent_raw_values`.
+No aggregate operation is needed and it should be set `raw_values`. The `valueKey` should be set to `recent_raw_values`.
 
 **Note**: `recent_raw_values` is filled only when the operation is `raw_values`.
 
@@ -540,7 +540,7 @@ The prometheus encoder specifies which metrics to export to prometheus and which
 For example, we may want to report the number of bytes and packets for the reported flows.
 For each reported metric, we may specify a different set of labels.
 Each metric may be renamed from its internal name.
-The internal metric name is specified as `valuekey` and the exported name is specified as `name`.
+The internal metric name is specified as `valueKey` and the exported name is specified as `name`.
 A prefix for all exported metrics may be specified, and this prefix is prepended to the `name` of each specified metric.
 
 ```yaml
@@ -554,14 +554,14 @@ parameters:
         metrics:
           - name: Bytes
             type: gauge
-            valuekey: bytes
+            valueKey: bytes
             labels:
               - srcAddr
               - dstAddr
               - srcPort
           - name: Packets
             type: counter
-            valuekey: packets
+            valueKey: packets
             labels:
               - srcAddr
               - dstAddr
@@ -632,6 +632,7 @@ Develop
   docs                  Update flowlogs-pipeline documentation  
   clean                 Clean  
   tests-unit            Unit tests  
+  tests-fast            Fast unit tests (no race tests / coverage)  
   tests-e2e             End-to-end tests  
   tests-all             All tests  
   benchmarks            Benchmark  

contrib/kubernetes/flowlogs-pipeline.conf.yaml

@@ -74,83 +74,96 @@ parameters:
       - input: dstIP
         output: dstLocation
         type: add_location
-        parameters: ""
     type: network
 - extract:
     aggregates:
-    - Name: bandwidth_network_service
-      By:
+    - name: bandwidth_network_service
+      by:
       - service
-      Operation: sum
-      RecordKey: bytes
-    - Name: bandwidth_source_destination_subnet
-      By:
+      operation: sum
+      recordKey: bytes
+      topK: 0
+    - name: bandwidth_source_destination_subnet
+      by:
       - dstSubnet24
       - srcSubnet24
-      Operation: sum
-      RecordKey: bytes
-    - Name: bandwidth_source_subnet
-      By:
+      operation: sum
+      recordKey: bytes
+      topK: 0
+    - name: bandwidth_source_subnet
+      by:
       - srcSubnet
-      Operation: sum
-      RecordKey: bytes
-    - Name: dest_connection_subnet_count
-      By:
+      operation: sum
+      recordKey: bytes
+      topK: 0
+    - name: dest_connection_subnet_count
+      by:
       - dstSubnet
-      Operation: sum
-      RecordKey: isNewFlow
-    - Name: src_connection_count
-      By:
+      operation: sum
+      recordKey: isNewFlow
+      topK: 0
+    - name: src_connection_count
+      by:
       - srcSubnet
-      Operation: count
-      RecordKey: ""
-    - Name: TCPFlags_count
-      By:
+      operation: count
+      recordKey: ""
+      topK: 0
+    - name: TCPFlags_count
+      by:
       - TCPFlags
-      Operation: count
-      RecordKey: ""
-    - Name: dst_as_connection_count
-      By:
+      operation: count
+      recordKey: ""
+      topK: 0
+    - name: dst_as_connection_count
+      by:
       - dstAS
-      Operation: count
-      RecordKey: ""
-    - Name: src_as_connection_count
-      By:
+      operation: count
+      recordKey: ""
+      topK: 0
+    - name: src_as_connection_count
+      by:
       - srcAS
-      Operation: count
-      RecordKey: ""
-    - Name: count_source_destination_subnet
-      By:
+      operation: count
+      recordKey: ""
+      topK: 0
+    - name: count_source_destination_subnet
+      by:
       - dstSubnet24
       - srcSubnet24
-      Operation: count
-      RecordKey: ""
-    - Name: bandwidth_destination_subnet
-      By:
+      operation: count
+      recordKey: ""
+      topK: 0
+    - name: bandwidth_destination_subnet
+      by:
       - dstSubnet
-      Operation: sum
-      RecordKey: bytes
-    - Name: bandwidth_namespace
-      By:
+      operation: sum
+      recordKey: bytes
+      topK: 0
+    - name: bandwidth_namespace
+      by:
       - srcK8S_Namespace
       - srcK8S_Type
-      Operation: sum
-      RecordKey: bytes
-    - Name: flows_bytes_hist
-      By:
+      operation: sum
+      recordKey: bytes
+      topK: 0
+    - name: flows_bytes_hist
+      by:
       - all_Evaluate
-      Operation: raw_values
-      RecordKey: bytes
-    - Name: dest_connection_location_count
-      By:
+      operation: raw_values
+      recordKey: bytes
+      topK: 0
+    - name: dest_connection_location_count
+      by:
       - dstLocation_CountryName
-      Operation: count
-      RecordKey: ""
-    - Name: dest_service_count
-      By:
+      operation: count
+      recordKey: ""
+      topK: 0
+    - name: dest_service_count
+      by:
       - service
-      Operation: count
-      RecordKey: ""
+      operation: count
+      recordKey: ""
+      topK: 0
     type: aggregates
   name: extract_aggregate
 - encode:
@@ -161,7 +174,7 @@ parameters:
         filter:
           key: name
           value: bandwidth_network_service
-        valuekey: recent_op_value
+        valueKey: recent_op_value
         labels:
         - by
         - aggregate
@@ -171,7 +184,7 @@ parameters:
         filter:
           key: name
           value: bandwidth_source_destination_subnet
-        valuekey: recent_op_value
+        valueKey: recent_op_value
         labels:
         - by
         - aggregate
@@ -181,7 +194,7 @@ parameters:
         filter:
           key: name
           value: bandwidth_source_subnet
-        valuekey: recent_op_value
+        valueKey: recent_op_value
         labels:
         - by
         - aggregate
@@ -191,7 +204,7 @@ parameters:
         filter:
           key: name
           value: dest_connection_subnet_count
-        valuekey: recent_count
+        valueKey: recent_count
         labels:
         - by
         - aggregate
@@ -201,7 +214,7 @@ parameters:
         filter:
           key: name
           value: src_connection_count
-        valuekey: recent_count
+        valueKey: recent_count
         labels:
         - by
         - aggregate
@@ -211,7 +224,7 @@ parameters:
         filter:
           key: name
           value: TCPFlags_count
-        valuekey: recent_count
+        valueKey: recent_count
         labels:
         - by
         - aggregate
@@ -221,7 +234,7 @@ parameters:
         filter:
           key: name
           value: dst_as_connection_count
-        valuekey: recent_count
+        valueKey: recent_count
         labels:
         - by
         - aggregate
@@ -231,7 +244,7 @@ parameters:
         filter:
           key: name
           value: src_as_connection_count
-        valuekey: recent_count
+        valueKey: recent_count
         labels:
         - by
         - aggregate
@@ -241,7 +254,7 @@ parameters:
         filter:
           key: name
           value: count_source_destination_subnet
-        valuekey: recent_count
+        valueKey: recent_count
         labels:
         - by
         - aggregate
@@ -251,7 +264,7 @@ parameters:
         filter:
           key: name
           value: bandwidth_destination_subnet
-        valuekey: recent_op_value
+        valueKey: recent_op_value
         labels:
         - by
         - aggregate
@@ -261,7 +274,7 @@ parameters:
         filter:
           key: name
           value: bandwidth_namespace
-        valuekey: recent_op_value
+        valueKey: recent_op_value
         labels:
         - by
         - aggregate
@@ -271,7 +284,7 @@ parameters:
         filter:
           key: name
           value: flows_bytes_hist
-        valuekey: recent_raw_values
+        valueKey: recent_raw_values
         labels:
         - by
         - aggregate
@@ -286,7 +299,7 @@ parameters:
         filter:
           key: name
           value: dest_connection_location_count
-        valuekey: recent_count
+        valueKey: recent_count
         labels:
         - by
         - aggregate
@@ -296,7 +309,7 @@ parameters:
         filter:
           key: name
           value: dest_service_count
-        valuekey: recent_count
+        valueKey: recent_count
         labels:
         - by
         - aggregate