Refactor(Tracing): Full rework (#126)

* refactor(tracing): combine archive and trace domain models, new optimal query

* fix(pagination): force timestamp microseconds in cursor

* feat(tracing): use proteced Q and scan start_time

* chore(api): update tracing OAS

* feat: show traces on pipeline page

* add cli command, fix migration, add ingress saving

* style: conform

go.mod

@@ -54,6 +54,7 @@ require (
 	github.com/containerd/containerd v1.7.14 // indirect
 	github.com/containerd/log v0.1.0 // indirect
 	github.com/cpuguy83/dockercfg v0.3.1 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/distribution/reference v0.5.0 // indirect
 	github.com/docker/go-connections v0.5.0 // indirect
@@ -119,16 +120,19 @@ require (
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/robfig/cron/v3 v3.0.1 // indirect
 	github.com/rs/xid v1.4.0 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/shirou/gopsutil/v3 v3.24.2 // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/tklauser/go-sysconf v0.3.13 // indirect
 	github.com/tklauser/numcpus v0.7.0 // indirect
 	github.com/ulikunitz/xz v0.5.9 // indirect
+	github.com/urfave/cli/v2 v2.27.5 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
+	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0 // indirect
 	go.opentelemetry.io/otel/sdk v1.32.0 // indirect

go.sum

@@ -38,6 +38,8 @@ github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3
 github.com/cpuguy83/dockercfg v0.3.1 h1:/FpZ+JaygUR/lZP2NlFI2DVfrOEMAIKP5wWEJdoYe9E=
 github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc=
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc=
+github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
 github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -266,6 +268,8 @@ github.com/rs/cors v1.8.2 h1:KCooALfAYGs415Cwu5ABvv9n9509fSiG5SQJn/AQo4U=
 github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
 github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY=
 github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
+github.com/russross/blackfriday v1.6.0 h1:KqfZb0pUVN2lYqZUYRddxF4OR8ZMURnJIG5Y3VRLtww=
+github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/samber/lo v1.39.0 h1:4gTz1wUhNYLhFSKl6O+8peW0v2F4BCY034GRpU9WnuA=
 github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA=
@@ -306,6 +310,8 @@ github.com/twpayne/go-geom v1.5.7/go.mod h1:y4fTAQtLedXW8eG2Yo4tYrIGN1yIwwKkmA+K
 github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 github.com/ulikunitz/xz v0.5.9 h1:RsKRIA2MO8x56wkkcd3LbtcE/uMszhb6DpRf+3uwa3I=
 github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
+github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w=
+github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasthttp v1.30.0/go.mod h1:2rsYD01CKFrjjsvFxx75KlEUNpWNBY9JWD3K/7o2Cus=
@@ -316,6 +322,8 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
 github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
 github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo=
 github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
+github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=
+github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=

internal/pagination/pagination.go

@@ -3,6 +3,7 @@ package pagination
 import (
 	"encoding/base64"
 	"fmt"
+	"log"
 	"net/http"
 	"net/url"
 	"reflect"
@@ -17,6 +18,19 @@ const (
 	DEFAULT_LIMIT = 100
 )
 
+var enc cbor.EncMode
+
+func init() {
+	var err error
+	encOpts := cbor.CoreDetEncOptions()
+	encOpts.Time = cbor.TimeUnixMicro
+	encOpts.TimeTag = cbor.EncTagRequired
+	enc, err = encOpts.EncMode()
+	if err != nil {
+		log.Fatalf("could not create cursor encoder: %s\n", err.Error())
+	}
+}
+
 type Links struct {
 	Previous string `json:"previous"`
 	Next     string `json:"next"`
@@ -90,11 +104,7 @@ func CreatePageT[T1 any, T2 any](data []T1, cursor Cursor[T2]) Page[T1] {
 }
 
 func EncodeCursor[T any](f Cursor[T]) string {
-	opt := cbor.CanonicalEncOptions()
-	opt.Time = cbor.TimeUnix
-	enc, _ := opt.EncMode()
-
-	data, err := enc.Marshal(&f)
+	data, err := enc.Marshal(f)
 	if err != nil {
 		panic(err)
 	}
@@ -156,8 +166,15 @@ func columnAlias(name string) string {
 	return "paginated_" + name
 }
 
+func ApplyNoLimit[T any](q sq.SelectBuilder, c Cursor[T]) (sq.SelectBuilder, error) {
+	c.Limit = 0
+	return Apply(q, c)
+}
+
 func Apply[T any](q sq.SelectBuilder, c Cursor[T]) (sq.SelectBuilder, error) {
-	q = q.Limit(c.Limit)
+	if c.Limit > 0 {
+		q = q.Limit(c.Limit)
+	}
 	rt := reflect.TypeOf(c.Columns)
 	rv := reflect.ValueOf(c.Columns)
 	columns := []whereCol{}

pkg/api/.openapi-generator/FILES

@@ -17,7 +17,6 @@ model_add_tenant_member_request.go
 model_api_error.go
 model_api_key.go
 model_api_key_created.go
-model_archived_ingress.go
 model_create_api_key_request.go
 model_create_device_201_response.go
 model_create_device_request.go
@@ -44,7 +43,6 @@ model_get_sensor_200_response.go
 model_get_sensor_group_200_response.go
 model_get_worker_200_response.go
 model_get_worker_user_code_200_response.go
-model_ingress_dto.go
 model_list_api_keys_200_response.go
 model_list_api_keys_200_response_all_of.go
 model_list_datastreams_200_response.go
@@ -53,8 +51,6 @@ model_list_device_sensors_200_response.go
 model_list_device_sensors_200_response_all_of.go
 model_list_devices_200_response.go
 model_list_devices_200_response_all_of.go
-model_list_ingresses_200_response.go
-model_list_ingresses_200_response_all_of.go
 model_list_pipelines_200_response.go
 model_list_pipelines_200_response_all_of.go
 model_list_sensor_groups_200_response.go
@@ -77,7 +73,6 @@ model_sensor.go
 model_sensor_group.go
 model_tenant.go
 model_trace.go
-model_trace_step.go
 model_update_device_200_response.go
 model_update_device_request.go
 model_update_pipeline_200_response.go

pkg/api/README.md

@@ -122,8 +122,7 @@ Class | Method | HTTP request | Description
 *TenantsApi* | [**ListTenants**](docs/TenantsApi.md#listtenants) | **Get** /tenants | Retrieves tenants
 *TenantsApi* | [**RemoveTenantMember**](docs/TenantsApi.md#removetenantmember) | **Delete** /tenants/{tenant_id}/members/{user_id} | Removes a member from a tenant
 *TenantsApi* | [**UpdateTenantMember**](docs/TenantsApi.md#updatetenantmember) | **Patch** /tenants/{tenant_id}/members/{user_id} | Update a tenant member's permissions
-*TracingApi* | [**ListIngresses**](docs/TracingApi.md#listingresses) | **Get** /ingresses | List ingresses
-*TracingApi* | [**ListTraces**](docs/TracingApi.md#listtraces) | **Get** /tracing | List traces
+*TracingApi* | [**ListTraces**](docs/TracingApi.md#listtraces) | **Get** /traces | List traces
 *UplinkApi* | [**ProcessUplinkData**](docs/UplinkApi.md#processuplinkdata) | **Post** /uplinks/{pipeline_id} | Process uplink message
 *WorkersApi* | [**CreateWorker**](docs/WorkersApi.md#createworker) | **Post** /workers | Create Worker
 *WorkersApi* | [**GetWorker**](docs/WorkersApi.md#getworker) | **Get** /workers/{id} | Get worker
@@ -141,7 +140,6 @@ Class | Method | HTTP request | Description
  - [ApiError](docs/ApiError.md)
  - [ApiKey](docs/ApiKey.md)
  - [ApiKeyCreated](docs/ApiKeyCreated.md)
- - [ArchivedIngress](docs/ArchivedIngress.md)
  - [CreateApiKeyRequest](docs/CreateApiKeyRequest.md)
  - [CreateDevice201Response](docs/CreateDevice201Response.md)
  - [CreateDeviceRequest](docs/CreateDeviceRequest.md)
@@ -168,7 +166,6 @@ Class | Method | HTTP request | Description
  - [GetSensorGroup200Response](docs/GetSensorGroup200Response.md)
  - [GetWorker200Response](docs/GetWorker200Response.md)
  - [GetWorkerUserCode200Response](docs/GetWorkerUserCode200Response.md)
- - [IngressDTO](docs/IngressDTO.md)
  - [ListApiKeys200Response](docs/ListApiKeys200Response.md)
  - [ListApiKeys200ResponseAllOf](docs/ListApiKeys200ResponseAllOf.md)
  - [ListDatastreams200Response](docs/ListDatastreams200Response.md)
@@ -177,8 +174,6 @@ Class | Method | HTTP request | Description
  - [ListDeviceSensors200ResponseAllOf](docs/ListDeviceSensors200ResponseAllOf.md)
  - [ListDevices200Response](docs/ListDevices200Response.md)
  - [ListDevices200ResponseAllOf](docs/ListDevices200ResponseAllOf.md)
- - [ListIngresses200Response](docs/ListIngresses200Response.md)
- - [ListIngresses200ResponseAllOf](docs/ListIngresses200ResponseAllOf.md)
  - [ListPipelines200Response](docs/ListPipelines200Response.md)
  - [ListPipelines200ResponseAllOf](docs/ListPipelines200ResponseAllOf.md)
  - [ListSensorGroups200Response](docs/ListSensorGroups200Response.md)
@@ -201,7 +196,6 @@ Class | Method | HTTP request | Description
  - [SensorGroup](docs/SensorGroup.md)
  - [Tenant](docs/Tenant.md)
  - [Trace](docs/Trace.md)
- - [TraceStep](docs/TraceStep.md)
  - [UpdateDevice200Response](docs/UpdateDevice200Response.md)
  - [UpdateDeviceRequest](docs/UpdateDeviceRequest.md)
  - [UpdatePipeline200Response](docs/UpdatePipeline200Response.md)