Fix deck gateway apply (#1508)

The `deck gateway apply` command added in v1.43.0 added additional HTTP calls to discover which functions are enabled. This does not work well when using an RBAC user with restricted permissions.

This change removes those additional checks and delegates the lookup of foreign keys for partial applications to `go-database-reconciler`

---------

Co-authored-by: Patryk Małek <[email protected]>

cmd/common.go

@@ -39,6 +39,13 @@ const (
 	modeLocal
 )
 
+type ApplyType int
+
+const (
+	ApplyTypeFull ApplyType = iota
+	ApplyTypePartial
+)
+
 var jsonOutput diff.JSONOutputObject
 
 func getMode(targetContent *file.Content) mode {
@@ -131,7 +138,7 @@ func RemoveConsumerPlugins(targetContentPlugins []file.FPlugin) []file.FPlugin {
 }
 
 func syncMain(ctx context.Context, filenames []string, dry bool, parallelism,
-	delay int, workspace string, enableJSONOutput bool, noDeletes bool,
+	delay int, workspace string, enableJSONOutput bool, applyType ApplyType,
 ) error {
 	// read target file
 	if enableJSONOutput {
@@ -157,11 +164,12 @@ func syncMain(ctx context.Context, filenames []string, dry bool, parallelism,
 	}
 
 	cmd := "sync"
-
-	isPartialApply := false
-	if noDeletes {
+	if applyType == ApplyTypePartial {
 		cmd = "apply"
-		isPartialApply = true
+		dumpConfig.IsPartialApply = true
+	} else {
+		// Explicitly set this here as dumpConfig is a global var
+		dumpConfig.IsPartialApply = false
 	}
 
 	if dry {
@@ -209,31 +217,14 @@ func syncMain(ctx context.Context, filenames []string, dry bool, parallelism,
 	// load Kong version after workspace
 	var kongVersion string
 	var parsedKongVersion semver.Version
-	isLicensedKongEnterprise := false
 	if mode == modeKonnect {
 		kongVersion = fetchKonnectKongVersion()
-		isLicensedKongEnterprise = true
 	} else {
 		kongVersion, err = fetchKongVersion(ctx, wsConfig)
 		if err != nil {
 			return fmt.Errorf("reading Kong version: %w", err)
 		}
-
-		// Are we running enterprise?
-		v, err := kong.ParseSemanticVersion(kongVersion)
-		if err != nil {
-			return fmt.Errorf("parsing Kong version: %w", err)
-		}
-
-		// Check if there's an active license for Consumer Group checks
-		if v.IsKongGatewayEnterprise() {
-			isLicensedKongEnterprise, err = isLicensed(ctx, wsConfig)
-			if err != nil {
-				return fmt.Errorf("checking if Kong is licensed: %w", err)
-			}
-		}
 	}
-
 	parsedKongVersion, err = reconcilerUtils.ParseKongVersion(kongVersion)
 	if err != nil {
 		return fmt.Errorf("parsing Kong version: %w", err)
@@ -272,24 +263,21 @@ func syncMain(ctx context.Context, filenames []string, dry bool, parallelism,
 		return err
 	}
 
-	// Consumer groups are an enterprise 3.4+ feature
-	if parsedKongVersion.GTE(reconcilerUtils.Kong340Version) && isLicensedKongEnterprise {
-		dumpConfig.LookUpSelectorTagsConsumerGroups, err = determineLookUpSelectorTagsConsumerGroups(*targetContent)
+	dumpConfig.LookUpSelectorTagsConsumerGroups, err = determineLookUpSelectorTagsConsumerGroups(*targetContent)
+	if err != nil {
+		return fmt.Errorf("error determining lookup selector tags for consumer groups: %w", err)
+	}
+
+	if dumpConfig.LookUpSelectorTagsConsumerGroups != nil {
+		consumerGroupsGlobal, err := dump.GetAllConsumerGroups(ctx, kongClient, dumpConfig.LookUpSelectorTagsConsumerGroups)
 		if err != nil {
-			return fmt.Errorf("error determining lookup selector tags for consumer groups: %w", err)
+			return fmt.Errorf("error retrieving global consumer groups via lookup selector tags: %w", err)
 		}
-
-		if dumpConfig.LookUpSelectorTagsConsumerGroups != nil || isPartialApply {
-			consumerGroupsGlobal, err := dump.GetAllConsumerGroups(ctx, kongClient, dumpConfig.LookUpSelectorTagsConsumerGroups)
+		for _, c := range consumerGroupsGlobal {
+			targetContent.ConsumerGroups = append(targetContent.ConsumerGroups,
+				file.FConsumerGroupObject{ConsumerGroup: *c.ConsumerGroup})
 			if err != nil {
-				return fmt.Errorf("error retrieving global consumer groups via lookup selector tags: %w", err)
-			}
-			for _, c := range consumerGroupsGlobal {
-				targetContent.ConsumerGroups = append(targetContent.ConsumerGroups,
-					file.FConsumerGroupObject{ConsumerGroup: *c.ConsumerGroup})
-				if err != nil {
-					return fmt.Errorf("error adding global consumer group %v: %w", *c.ConsumerGroup.Name, err)
-				}
+				return fmt.Errorf("error adding global consumer group %v: %w", *c.ConsumerGroup.Name, err)
 			}
 		}
 	}
@@ -299,7 +287,7 @@ func syncMain(ctx context.Context, filenames []string, dry bool, parallelism,
 		return fmt.Errorf("error determining lookup selector tags for consumers: %w", err)
 	}
 
-	if dumpConfig.LookUpSelectorTagsConsumers != nil || isPartialApply {
+	if dumpConfig.LookUpSelectorTagsConsumers != nil {
 		consumersGlobal, err := dump.GetAllConsumers(ctx, kongClient, dumpConfig.LookUpSelectorTagsConsumers)
 		if err != nil {
 			return fmt.Errorf("error retrieving global consumers via lookup selector tags: %w", err)
@@ -317,7 +305,7 @@ func syncMain(ctx context.Context, filenames []string, dry bool, parallelism,
 		return fmt.Errorf("error determining lookup selector tags for routes: %w", err)
 	}
 
-	if dumpConfig.LookUpSelectorTagsRoutes != nil || isPartialApply {
+	if dumpConfig.LookUpSelectorTagsRoutes != nil {
 		routesGlobal, err := dump.GetAllRoutes(ctx, kongClient, dumpConfig.LookUpSelectorTagsRoutes)
 		if err != nil {
 			return fmt.Errorf("error retrieving global routes via lookup selector tags: %w", err)
@@ -335,7 +323,7 @@ func syncMain(ctx context.Context, filenames []string, dry bool, parallelism,
 		return fmt.Errorf("error determining lookup selector tags for services: %w", err)
 	}
 
-	if dumpConfig.LookUpSelectorTagsServices != nil || isPartialApply {
+	if dumpConfig.LookUpSelectorTagsServices != nil {
 		servicesGlobal, err := dump.GetAllServices(ctx, kongClient, dumpConfig.LookUpSelectorTagsServices)
 		if err != nil {
 			return fmt.Errorf("error retrieving global services via lookup selector tags: %w", err)
@@ -400,7 +388,9 @@ func syncMain(ctx context.Context, filenames []string, dry bool, parallelism,
 	}
 
 	totalOps, err := performDiff(
-		ctx, currentState, targetState, dry, parallelism, delay, kongClient, mode == modeKonnect, enableJSONOutput, noDeletes)
+		ctx, currentState, targetState, dry, parallelism, delay, kongClient, mode == modeKonnect,
+		enableJSONOutput, applyType,
+	)
 	if err != nil {
 		if enableJSONOutput {
 			var errs reconcilerUtils.ErrArray
@@ -529,16 +519,18 @@ func fetchCurrentState(ctx context.Context, client *kong.Client, dumpConfig dump
 
 func performDiff(ctx context.Context, currentState, targetState *state.KongState,
 	dry bool, parallelism int, delay int, client *kong.Client, isKonnect bool,
-	enableJSONOutput bool, noDeletes bool,
+	enableJSONOutput bool, applyType ApplyType,
 ) (int, error) {
+	shouldSkipDeletes := applyType == ApplyTypePartial
+
 	s, err := diff.NewSyncer(diff.SyncerOpts{
 		CurrentState:  currentState,
 		TargetState:   targetState,
 		KongClient:    client,
 		StageDelaySec: delay,
 		NoMaskValues:  noMaskValues,
 		IsKonnect:     isKonnect,
-		NoDeletes:     noDeletes,
+		NoDeletes:     shouldSkipDeletes,
 	})
 	if err != nil {
 		return 0, err
@@ -570,31 +562,6 @@ func performDiff(ctx context.Context, currentState, targetState *state.KongState
 	return int(totalOps), nil
 }
 
-func isLicensed(ctx context.Context, config reconcilerUtils.KongClientConfig) (bool, error) {
-	client, err := reconcilerUtils.GetKongClient(config)
-	if err != nil {
-		return false, err
-	}
-
-	req, err := http.NewRequest("GET",
-		reconcilerUtils.CleanAddress(config.Address)+"/",
-		nil)
-	if err != nil {
-		return false, err
-	}
-	var resp map[string]interface{}
-	_, err = client.Do(ctx, req, &resp)
-	if err != nil {
-		return false, err
-	}
-	_, ok := resp["license"]
-	if !ok {
-		return false, nil
-	}
-
-	return true, nil
-}
-
 func fetchKongVersion(ctx context.Context, config reconcilerUtils.KongClientConfig) (string, error) {
 	var version string
 

cmd/common_konnect.go

@@ -127,7 +127,7 @@ func resetKonnectV2(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	_, err = performDiff(ctx, currentState, targetState, false, 10, 0, client, true, resetJSONOutput, false)
+	_, err = performDiff(ctx, currentState, targetState, false, 10, 0, client, true, resetJSONOutput, ApplyTypeFull)
 	if err != nil {
 		return err
 	}
@@ -247,6 +247,7 @@ func syncKonnect(ctx context.Context,
 	}
 
 	stats, errs, _ := s.Solve(ctx, parallelism, dry, false)
+
 	// print stats before error to report completed operations
 	printStats(stats)
 	if errs != nil {

cmd/gateway_apply.go

@@ -15,7 +15,7 @@ var applyCmdKongStateFile []string
 
 func executeApply(cmd *cobra.Command, _ []string) error {
 	return syncMain(cmd.Context(), applyCmdKongStateFile, false,
-		applyCmdParallelism, applyCmdDBUpdateDelay, applyWorkspace, applyJSONOutput, true)
+		applyCmdParallelism, applyCmdDBUpdateDelay, applyWorkspace, applyJSONOutput, ApplyTypePartial)
 }
 
 func newApplyCmd() *cobra.Command {

cmd/gateway_diff.go

@@ -17,7 +17,7 @@ var (
 
 func executeDiff(cmd *cobra.Command, _ []string) error {
 	return syncMain(cmd.Context(), diffCmdKongStateFile, true,
-		diffCmdParallelism, 0, diffWorkspace, diffJSONOutput, false)
+		diffCmdParallelism, 0, diffWorkspace, diffJSONOutput, ApplyTypeFull)
 }
 
 // newDiffCmd represents the diff command

cmd/gateway_reset.go

@@ -97,7 +97,7 @@ func executeReset(cmd *cobra.Command, _ []string) error {
 		if err != nil {
 			return err
 		}
-		_, err = performDiff(ctx, currentState, targetState, false, 10, 0, wsClient, false, resetJSONOutput, false)
+		_, err = performDiff(ctx, currentState, targetState, false, 10, 0, wsClient, false, resetJSONOutput, ApplyTypeFull)
 		if err != nil {
 			return err
 		}

cmd/gateway_sync.go

@@ -18,7 +18,7 @@ var syncCmdKongStateFile []string
 
 func executeSync(cmd *cobra.Command, _ []string) error {
 	return syncMain(cmd.Context(), syncCmdKongStateFile, false,
-		syncCmdParallelism, syncCmdDBUpdateDelay, syncWorkspace, syncJSONOutput, false)
+		syncCmdParallelism, syncCmdDBUpdateDelay, syncWorkspace, syncJSONOutput, ApplyTypeFull)
 }
 
 // newSyncCmd represents the sync command

go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/fatih/color v1.17.0
 	github.com/google/go-cmp v0.6.0
 	github.com/kong/go-apiops v0.1.41
-	github.com/kong/go-database-reconciler v1.18.1
+	github.com/kong/go-database-reconciler v1.19.2
 	github.com/kong/go-kong v0.63.0
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/spf13/cobra v1.8.1

go.sum

@@ -249,6 +249,8 @@ github.com/kong/go-apiops v0.1.41 h1:1KXbQqyhO2E4nEnXJNqUDmHZU/LQ1U7Zoi+MAlcM2P0
 github.com/kong/go-apiops v0.1.41/go.mod h1:sATq9Tz+ivzHKZU+tDXkRtEZnf64xroU3lgv3yXqP4I=
 github.com/kong/go-database-reconciler v1.18.1 h1:NhV3oDWMO02yj9scFIYfUwoK/KxMxCP0fXkZIBF1QME=
 github.com/kong/go-database-reconciler v1.18.1/go.mod h1:BqaV17xmjYAJfQYlaMKNz/DbJO4FfpwZpKD5dg2Pku0=
+github.com/kong/go-database-reconciler v1.19.2 h1:+TA5fs5BJWrX3FpsuekIitCqW66M/rw5ToRoUZTLxdA=
+github.com/kong/go-database-reconciler v1.19.2/go.mod h1:fSzg8w4rBaiMqF0H9XqoGXCJsqLIxRBG9f1BhvKU2Lg=
 github.com/kong/go-kong v0.63.0 h1:8ECLgkgDqON61qCMq/M0gTwZKYxg55Oy692dRDOOBiU=
 github.com/kong/go-kong v0.63.0/go.mod h1:ma9GWnhkxtrXZlLFfED955HjVzmUojYEHet3lm+PDik=
 github.com/kong/go-slugify v1.0.0 h1:vCFAyf2sdoSlBtLcrmDWUFn0ohlpKiKvQfXZkO5vSKY=

tests/integration/apply_test.go

@@ -48,6 +48,20 @@ func Test_Apply_3x(t *testing.T) {
 			expectedState: "testdata/apply/004-foreign-keys-consumer-groups/expected-state.yaml",
 			runWhen:       "enterprise",
 		},
+		{
+			name:          "accepts service foreign keys",
+			firstFile:     "testdata/apply/005-foreign-keys-services/service-01.yaml",
+			secondFile:    "testdata/apply/005-foreign-keys-services/plugin-01.yaml",
+			expectedState: "testdata/apply/005-foreign-keys-services/expected-state.yaml",
+			runWhen:       "kong",
+		},
+		{
+			name:          "accepts route foreign keys",
+			firstFile:     "testdata/apply/006-foreign-keys-routes/route-01.yaml",
+			secondFile:    "testdata/apply/006-foreign-keys-routes/plugin-01.yaml",
+			expectedState: "testdata/apply/006-foreign-keys-routes/expected-state.yaml",
+			runWhen:       "kong",
+		},
 	}
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
@@ -67,4 +81,29 @@ func Test_Apply_3x(t *testing.T) {
 			assert.Equal(t, expected, out)
 		})
 	}
+
+	t.Run("updates existing entities", func(t *testing.T) {
+		runWhen(t, "kong", ">=3.0.0")
+		setup(t)
+
+		err := apply(context.Background(), "testdata/apply/007-update-existing-entity/service-01.yaml")
+		require.NoError(t, err, "failed to apply service-01")
+
+		out, err := dump()
+		require.NoError(t, err)
+		expectedOriginal, err := readFile("testdata/apply/007-update-existing-entity/expected-state-01.yaml")
+		require.NoError(t, err, "failed to read expected state")
+
+		assert.Equal(t, expectedOriginal, out)
+
+		err = apply(context.Background(), "testdata/apply/007-update-existing-entity/service-02.yaml")
+		require.NoError(t, err, "failed to apply service-02")
+
+		expectedChanged, err := readFile("testdata/apply/007-update-existing-entity/expected-state-02.yaml")
+		require.NoError(t, err, "failed to read expected state")
+
+		outChanged, err := dump()
+		require.NoError(t, err)
+		assert.Equal(t, expectedChanged, outChanged)
+	})
 }

tests/integration/testdata/apply/005-foreign-keys-services/expected-state.yaml

@@ -0,0 +1,24 @@
+_format_version: "3.0"
+services:
+- connect_timeout: 60000
+  enabled: true
+  host: httpbin.konghq.com
+  name: example-service
+  plugins:
+  - config:
+      body: null
+      content_type: null
+      echo: false
+      message: null
+      status_code: 404
+      trigger: null
+    enabled: true
+    name: request-termination
+    protocols:
+    - http
+    - https
+  port: 80
+  protocol: http
+  read_timeout: 60000
+  retries: 5
+  write_timeout: 60000

tests/integration/testdata/apply/005-foreign-keys-services/plugin-01.yaml

@@ -0,0 +1,15 @@
+_format_version: "3.0"
+plugins:
+  - name: request-termination
+    enabled: true
+    service: example-service
+    config:
+      status_code: 404
+      body: null
+      content_type: null
+      echo: false
+      message: null
+      trigger: null
+    protocols:
+      - http
+      - https

tests/integration/testdata/apply/005-foreign-keys-services/service-01.yaml

@@ -0,0 +1,4 @@
+_format_version: "3.0"
+services:
+  - name: example-service
+    url: http://httpbin.konghq.com

tests/integration/testdata/apply/006-foreign-keys-routes/expected-state.yaml

@@ -0,0 +1,28 @@
+_format_version: "3.0"
+routes:
+- https_redirect_status_code: 426
+  name: example-route
+  path_handling: v0
+  paths:
+  - /mock
+  plugins:
+  - config:
+      body: null
+      content_type: null
+      echo: false
+      message: null
+      status_code: 404
+      trigger: null
+    enabled: true
+    name: request-termination
+    protocols:
+    - http
+    - https
+  preserve_host: false
+  protocols:
+  - http
+  - https
+  regex_priority: 0
+  request_buffering: true
+  response_buffering: true
+  strip_path: true