Upgrade builtin connectors, SDK and commons (#1515)

* upgrade builtin connectors

* rename deprecated parameter

* fix metadata constants test

* make generate

* clone config map when sending it to builtin plugin

.golangci.yml

@@ -1,7 +1,7 @@
 run:
   timeout: 3m
-  skip-dirs-use-default: false
-  skip-dirs:
+  exclude-dirs-use-default: false
+  exclude-dirs:
     - ^examples/
     - ^ui/
     - ^pkg/plugin/processor/builtin/internal/diff # external code

go.mod

@@ -8,15 +8,15 @@ require (
 	github.com/Masterminds/sprig/v3 v3.2.3
 	github.com/NYTimes/gziphandler v1.1.1
 	github.com/bufbuild/buf v1.31.0
-	github.com/conduitio/conduit-commons v0.1.1
+	github.com/conduitio/conduit-commons v0.2.0
 	github.com/conduitio/conduit-connector-file v0.6.0
-	github.com/conduitio/conduit-connector-generator v0.5.0
-	github.com/conduitio/conduit-connector-kafka v0.7.2
+	github.com/conduitio/conduit-connector-generator v0.6.0
+	github.com/conduitio/conduit-connector-kafka v0.8.0
 	github.com/conduitio/conduit-connector-log v0.3.0
-	github.com/conduitio/conduit-connector-postgres v0.6.0
-	github.com/conduitio/conduit-connector-protocol v0.5.1-0.20240104160905-e9e61586fb8d
+	github.com/conduitio/conduit-connector-postgres v0.7.0
+	github.com/conduitio/conduit-connector-protocol v0.6.0
 	github.com/conduitio/conduit-connector-s3 v0.5.1
-	github.com/conduitio/conduit-connector-sdk v0.8.0
+	github.com/conduitio/conduit-connector-sdk v0.9.1
 	github.com/conduitio/conduit-processor-sdk v0.1.1
 	github.com/conduitio/yaml/v3 v3.3.0
 	github.com/dgraph-io/badger/v4 v4.2.0
@@ -124,7 +124,7 @@ require (
 	github.com/containerd/stargz-snapshotter/estargz v0.15.1 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
 	github.com/curioswitch/go-reassign v0.2.0 // indirect
-	github.com/daixiang0/gci v0.12.3 // indirect
+	github.com/daixiang0/gci v0.13.4 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/denis-tingaikin/go-header v0.5.0 // indirect
 	github.com/dgraph-io/ristretto v0.1.1 // indirect

go.sum

@@ -327,24 +327,24 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH
 github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
 github.com/colinmarc/hdfs/v2 v2.1.1/go.mod h1:M3x+k8UKKmxtFu++uAZ0OtDU8jR3jnaZIAc6yK4Ue0c=
-github.com/conduitio/conduit-commons v0.1.1 h1:ACgRexA85QIme5NE5AzrsO48RcqIBNSmwdBjWgXjxbg=
-github.com/conduitio/conduit-commons v0.1.1/go.mod h1:ejihG+J5Q6V3pucBPKlsdw3Kfzw0ggONn1SIm1RLzTc=
+github.com/conduitio/conduit-commons v0.2.0 h1:TMpVGXi0Wski537qLAyQWdGjuGHEhaZxOS5L90pZJSQ=
+github.com/conduitio/conduit-commons v0.2.0/go.mod h1:i7Q2jm7FBSi2zj1/4MCsFD1hIKAbvamlNtSQfkhUTiY=
 github.com/conduitio/conduit-connector-file v0.6.0 h1:8tsGeGhKvFwYQZztOOL5/tmOhVShsfo9lQ3b/0fX8kQ=
 github.com/conduitio/conduit-connector-file v0.6.0/go.mod h1:ju7PiB4kTJgqng4KVXDt/Gvw/53kFwSzi5Ez9EDXxNI=
-github.com/conduitio/conduit-connector-generator v0.5.0 h1:zpXHif89DCJ13nftKLv31uI2AJGicpY5H1V7SwldRNo=
-github.com/conduitio/conduit-connector-generator v0.5.0/go.mod h1:CQKGYQNAnO6+bJSurzwtmGf98XoyeU6zsMJI/OCrlJI=
-github.com/conduitio/conduit-connector-kafka v0.7.2 h1:2kiCYI6CbuSGdNGBbhtjPB4Zv5PjtFduf35EL6vv0gY=
-github.com/conduitio/conduit-connector-kafka v0.7.2/go.mod h1:dpkglwPd42tdgUJpYEFWM2WOLCwFDP9HC5EPqMuiVUU=
+github.com/conduitio/conduit-connector-generator v0.6.0 h1:GjeR3v+mVGqXfGXs7zJMOHJFsX/SUpY7x4h1UhxpMzk=
+github.com/conduitio/conduit-connector-generator v0.6.0/go.mod h1:VYJjGhPh3N8ka5tr7a/oPyGhfrVGxr0MjCS+ccfteDM=
+github.com/conduitio/conduit-connector-kafka v0.8.0 h1:NsiihUjhzl+PWrxTWrbtCEV6UDfvLiEEuv7BaABr8zc=
+github.com/conduitio/conduit-connector-kafka v0.8.0/go.mod h1:1fkIxtjojlTKNQ0RpMNPUkz2h1VNVtZHOyIsZVT1U2I=
 github.com/conduitio/conduit-connector-log v0.3.0 h1:J9CD/y86D/3i/C1MI0lrTiFnLwwnpJKQJu3M+jDiI4Y=
 github.com/conduitio/conduit-connector-log v0.3.0/go.mod h1:R0gHB21mw/9BXYAKfx9IKQLEsPuB05zJPJXB6JFxYGs=
-github.com/conduitio/conduit-connector-postgres v0.6.0 h1:D0YTZXkxm5FG/UvL80MOP59RqozaFXKwr7sGII6fmn4=
-github.com/conduitio/conduit-connector-postgres v0.6.0/go.mod h1:KASWjEASlHhMPV6xQCKMzP4qH2JRGarAto5AeLnBiCk=
-github.com/conduitio/conduit-connector-protocol v0.5.1-0.20240104160905-e9e61586fb8d h1:XkJkS63Qkf2vuIYZnWeQMOKsMxBCvqlhX+T+aDqh0kc=
-github.com/conduitio/conduit-connector-protocol v0.5.1-0.20240104160905-e9e61586fb8d/go.mod h1:zW3my/7U6dBCBwXZt4EdtmLHBUhhX/42fXMu/+4eRHU=
+github.com/conduitio/conduit-connector-postgres v0.7.0 h1:At6O5e4eFegESTgzxZeY1eh0i+1FO5L4R35PZ9s7te0=
+github.com/conduitio/conduit-connector-postgres v0.7.0/go.mod h1:d39rdzjiCFUaBbFsg+Lh0KorMIY6KbQDu8HrDu/5Zvs=
+github.com/conduitio/conduit-connector-protocol v0.6.0 h1:2gMOCOpa+c97CHIpZv7Niu3V4o5UgRr6fzj9kzfRV7o=
+github.com/conduitio/conduit-connector-protocol v0.6.0/go.mod h1:3mo59xYX9etFoR3n82R7J50La1iWK+Vm63H8z2wo4QM=
 github.com/conduitio/conduit-connector-s3 v0.5.1 h1:yRo8004ryCIZc/S3iWQ1rN6pm6bjySlXFCGZOl1rE1E=
 github.com/conduitio/conduit-connector-s3 v0.5.1/go.mod h1:nbxzsyS95gbFJ28Job9vFFB+byRFINSv70/13Yi4mKQ=
-github.com/conduitio/conduit-connector-sdk v0.8.0 h1:gvchqoj5d3AQsBoIosx4i32L8Ex9+5BuAyHi/IM9VD4=
-github.com/conduitio/conduit-connector-sdk v0.8.0/go.mod h1:nOz4K3X6fD8YMe5CPbULwSEE18Eu02ZrpT6o6KwQfxs=
+github.com/conduitio/conduit-connector-sdk v0.9.1 h1:DiMUn7udnjWvyaDsyeTZFHeYTEIdqUU6dqPunEEE3Kw=
+github.com/conduitio/conduit-connector-sdk v0.9.1/go.mod h1:cNoofumgDlsaThkxkNYg7zab4AkmRZt1V711aO7guGU=
 github.com/conduitio/conduit-processor-sdk v0.1.1 h1:C+5Z9pGKVTpdIf5QFNx4UxpvxuOylGRVkGidEpom7HQ=
 github.com/conduitio/conduit-processor-sdk v0.1.1/go.mod h1:StkbqQX1WxTjr9LOy7zY+e3DAbEDVvozeamELdzFqck=
 github.com/conduitio/yaml/v3 v3.3.0 h1:kbbaOSHcuH39gP4+rgbJGl6DSbLZcJgEaBvkEXJlCsI=
@@ -366,8 +366,8 @@ github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
 github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
 github.com/curioswitch/go-reassign v0.2.0 h1:G9UZyOcpk/d7Gd6mqYgd8XYWFMw/znxwGDUstnC9DIo=
 github.com/curioswitch/go-reassign v0.2.0/go.mod h1:x6OpXuWvgfQaMGks2BZybTngWjT84hqJfKoO8Tt/Roc=
-github.com/daixiang0/gci v0.12.3 h1:yOZI7VAxAGPQmkb1eqt5g/11SUlwoat1fSblGLmdiQc=
-github.com/daixiang0/gci v0.12.3/go.mod h1:xtHP9N7AHdNvtRNfcx9gwTDfw7FRJx4bZUsiEfiNNAI=
+github.com/daixiang0/gci v0.13.4 h1:61UGkmpoAcxHM2hhNkZEf5SzwQtWJXTSws7jaPyqwlw=
+github.com/daixiang0/gci v0.13.4/go.mod h1:12etP2OniiIdP4q+kjUGrC/rUagga7ODbqsom5Eo5Yk=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -561,7 +561,6 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt
 github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
 github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
-github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
 github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
 github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=

pkg/plugin/connector/builtin/v1/internal/toplugin/destination.go

@@ -15,6 +15,8 @@
 package toplugin
 
 import (
+	"maps"
+
 	"github.com/conduitio/conduit-connector-protocol/cpluginv1"
 	"github.com/conduitio/conduit/pkg/record"
 )
@@ -23,7 +25,7 @@ func DestinationConfigureRequest(in map[string]string) cpluginv1.DestinationConf
 	out := cpluginv1.DestinationConfigureRequest{}
 	if len(in) > 0 {
 		// gRPC sends `nil` if the map is empty, match behavior
-		out.Config = in
+		out.Config = maps.Clone(in)
 	}
 	return out
 }

pkg/plugin/connector/builtin/v1/internal/toplugin/source.go

@@ -15,6 +15,8 @@
 package toplugin
 
 import (
+	"maps"
+
 	"github.com/conduitio/conduit-connector-protocol/cpluginv1"
 	"github.com/conduitio/conduit/pkg/record"
 )
@@ -23,7 +25,7 @@ func SourceConfigureRequest(in map[string]string) cpluginv1.SourceConfigureReque
 	out := cpluginv1.SourceConfigureRequest{}
 	if len(in) > 0 {
 		// gRPC sends `nil` if the map is empty, match behavior
-		out.Config = in
+		out.Config = maps.Clone(in)
 	}
 	return out
 }

pkg/plugin/connector/standalone/v1/metadata_test.go

@@ -17,8 +17,8 @@ package standalonev1
 import (
 	"testing"
 
+	metadatav1 "github.com/conduitio/conduit-commons/proto/metadata/v1"
 	opencdcv1 "github.com/conduitio/conduit-commons/proto/opencdc/v1"
-	connectorv1 "github.com/conduitio/conduit-connector-protocol/proto/connector/v1"
 	"github.com/conduitio/conduit/pkg/record"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/runtime/protoimpl"
@@ -30,10 +30,10 @@ func TestMetadataConstants(t *testing.T) {
 		record.MetadataOpenCDCVersion:                  opencdcv1.E_MetadataVersion,
 		record.MetadataCreatedAt:                       opencdcv1.E_MetadataCreatedAt,
 		record.MetadataReadAt:                          opencdcv1.E_MetadataReadAt,
-		record.MetadataConduitSourcePluginName:         connectorv1.E_MetadataConduitSourcePluginName,
-		record.MetadataConduitSourcePluginVersion:      connectorv1.E_MetadataConduitSourcePluginVersion,
-		record.MetadataConduitDestinationPluginName:    connectorv1.E_MetadataConduitDestinationPluginName,
-		record.MetadataConduitDestinationPluginVersion: connectorv1.E_MetadataConduitDestinationPluginVersion,
+		record.MetadataConduitSourcePluginName:         metadatav1.E_MetadataConduitSourcePluginName,
+		record.MetadataConduitSourcePluginVersion:      metadatav1.E_MetadataConduitSourcePluginVersion,
+		record.MetadataConduitDestinationPluginName:    metadatav1.E_MetadataConduitDestinationPluginName,
+		record.MetadataConduitDestinationPluginVersion: metadatav1.E_MetadataConduitDestinationPluginVersion,
 	}
 	for goConstant, extensionInfo := range wantMapping {
 		protoConstant := proto.GetExtension(extensionInfo.TypeDescriptor().ParentFile().Options(), extensionInfo)

pkg/plugin/processor/builtin/internal/exampleutil/example.go

@@ -20,10 +20,9 @@ import (
 	"fmt"
 	"log"
 
-	"github.com/conduitio/conduit/pkg/plugin/processor/builtin/internal"
-
 	"github.com/conduitio/conduit-commons/opencdc"
 	sdk "github.com/conduitio/conduit-processor-sdk"
+	"github.com/conduitio/conduit/pkg/plugin/processor/builtin/internal"
 	"github.com/conduitio/conduit/pkg/plugin/processor/builtin/internal/diff"
 	"github.com/goccy/go-json"
 	"github.com/google/go-cmp/cmp"

pkg/plugin/processor/builtin/internal/exampleutil/export.go

@@ -23,9 +23,8 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/conduitio/conduit/pkg/foundation/cerrors"
-
 	"github.com/conduitio/conduit-commons/opencdc"
+	"github.com/conduitio/conduit/pkg/foundation/cerrors"
 	"github.com/goccy/go-json"
 )
 

pkg/plugin/processor/builtin/internal/exampleutil/specs/avro.decode.json

@@ -60,12 +60,7 @@
       "summary": "Decode a record field in Avro format",
       "description": "This example shows the usage of the `avro.decode` processor.\nThe processor decodes the record's`.Key` field using the schema that is\ndownloaded from the schema registry and needs to exist under the subject`example-decode`.\nIn this example we use the following schema:\n\n```json\n{\n  \"type\":\"record\",\n  \"name\":\"record\",\n  \"fields\":[\n    {\"name\":\"myString\",\"type\":\"string\"},\n    {\"name\":\"myInt\",\"type\":\"int\"}\n  ]\n}\n```",
       "config": {
-        "auth.basic.password": "",
-        "auth.basic.username": "",
         "field": ".Key",
-        "tls.ca.cert": "",
-        "tls.client.cert": "",
-        "tls.client.key": "",
         "url": "http://127.0.0.1:54322"
       },
       "have": {

pkg/plugin/processor/builtin/internal/exampleutil/specs/avro.encode.json

@@ -98,16 +98,9 @@
       "summary": "Auto-register schema",
       "description": "This example shows the usage of the `avro.encode` processor\nwith the `autoRegister` schema strategy. The processor encodes the record's\n`.Payload.After` field using the schema that is extracted from the data\nand registered on the fly under the subject `example-autoRegister`.",
       "config": {
-        "auth.basic.password": "",
-        "auth.basic.username": "",
         "field": ".Payload.After",
         "schema.autoRegister.subject": "example-autoRegister",
-        "schema.preRegistered.subject": "",
-        "schema.preRegistered.version": "",
         "schema.strategy": "autoRegister",
-        "tls.ca.cert": "",
-        "tls.client.cert": "",
-        "tls.client.key": "",
         "url": "http://127.0.0.1:54322"
       },
       "have": {
@@ -151,16 +144,10 @@
       "summary": "Pre-register schema",
       "description": "This example shows the usage of the `avro.encode` processor\nwith the `preRegistered` schema strategy. When using this strategy, the\nschema has to be manually pre-registered. In this example we use the following schema:\n\n```json\n{\n  \"type\":\"record\",\n  \"name\":\"record\",\n  \"fields\":[\n    {\"name\":\"myString\",\"type\":\"string\"},\n    {\"name\":\"myInt\",\"type\":\"int\"}\n  ]\n}\n```\n\nThe processor encodes the record's`.Key` field using the above schema.",
       "config": {
-        "auth.basic.password": "",
-        "auth.basic.username": "",
         "field": ".Key",
-        "schema.autoRegister.subject": "",
         "schema.preRegistered.subject": "example-preRegistered",
         "schema.preRegistered.version": "1",
         "schema.strategy": "preRegistered",
-        "tls.ca.cert": "",
-        "tls.client.cert": "",
-        "tls.client.key": "",
         "url": "http://127.0.0.1:54322"
       },
       "have": {

pkg/plugin/processor/builtin/internal/exampleutil/specs/custom.javascript.json

@@ -25,8 +25,7 @@
       "summary": "Modify a record's metadata and payload using JavaScript",
       "description": "In this example we use the `custom.javascript` processor to add a metadata key to the input record. It also prepends \"hello, \" to `.Payload.After`.",
       "config": {
-        "script": "function process(rec) {\n  rec.Metadata[\"processed\"] = \"true\";\n  let existing = String.fromCharCode.apply(String, rec.Payload.After);\n  rec.Payload.After = RawData(\"hello, \" + existing);\n  return rec;\n}",
-        "script.path": ""
+        "script": "function process(rec) {\n  rec.Metadata[\"processed\"] = \"true\";\n  let existing = String.fromCharCode.apply(String, rec.Payload.After);\n  rec.Payload.After = RawData(\"hello, \" + existing);\n  return rec;\n}"
       },
       "have": {
         "position": null,

pkg/plugin/processor/builtin/internal/exampleutil/specs/webhook.http.json

@@ -130,12 +130,10 @@
         "backoffRetry.factor": "2",
         "backoffRetry.max": "5s",
         "backoffRetry.min": "100ms",
-        "request.body": "",
         "request.contentType": "application/json",
         "request.method": "GET",
         "request.url": "http://127.0.0.1:54321/{{.Payload.After.name}}",
-        "response.body": ".Payload.After.response",
-        "response.status": ""
+        "response.body": ".Payload.After.response"
       },
       "have": {
         "position": "cG9zLTE=",

pkg/processor/runnable_processor.go

@@ -16,6 +16,7 @@ package processor
 
 import (
 	"context"
+	"maps"
 
 	"github.com/conduitio/conduit-commons/opencdc"
 	sdk "github.com/conduitio/conduit-processor-sdk"
@@ -44,7 +45,7 @@ func newRunnableProcessor(
 }
 
 func (p *RunnableProcessor) Open(ctx context.Context) error {
-	err := p.proc.Configure(ctx, p.Config.Settings)
+	err := p.proc.Configure(ctx, maps.Clone(p.Config.Settings))
 	if err != nil {
 		return cerrors.Errorf("failed configuring processor: %w", err)
 	}