feat(cp): add dp metadata on prem (KAG-2444) (#11625)

* feat(cp): add dp metadata on prem (KAG-2444)
* add tests for older DPs

---------

Co-authored-by: Rob Serafini <[email protected]>

changelog/unreleased/kong/11625.yml

@@ -0,0 +1,7 @@
+"message": "**Clustering**: Allow configuring DP metadata labels for on-premise CP Gateway"
+"type": "feature"
+"scope": "Clustering"
+"prs":
+- 11625
+"jiras":
+- "KAG-2444"

kong-3.5.0-0.rockspec

@@ -261,6 +261,7 @@ build = {
     ["kong.db.migrations.core.018_310_to_320"] = "kong/db/migrations/core/018_310_to_320.lua",
     ["kong.db.migrations.core.019_320_to_330"] = "kong/db/migrations/core/019_320_to_330.lua",
     ["kong.db.migrations.core.020_330_to_340"] = "kong/db/migrations/core/020_330_to_340.lua",
+    ["kong.db.migrations.core.021_340_to_350"] = "kong/db/migrations/core/021_340_to_350.lua",
     ["kong.db.migrations.operations.200_to_210"] = "kong/db/migrations/operations/200_to_210.lua",
     ["kong.db.migrations.operations.212_to_213"] = "kong/db/migrations/operations/212_to_213.lua",
     ["kong.db.migrations.operations.280_to_300"] = "kong/db/migrations/operations/280_to_300.lua",

kong/clustering/control_plane.lua

@@ -234,6 +234,7 @@ function _M:handle_cp_websocket()
       ip = dp_ip,
       version = dp_version,
       sync_status = sync_status, -- TODO: import may have been failed though
+      labels = data.labels,
     }, { ttl = purge_delay })
     if not ok then
       ngx_log(ngx_ERR, _log_prefix, "unable to update clustering data plane status: ", err, log_suffix)

kong/db/migrations/core/021_340_to_350.lua

@@ -0,0 +1,13 @@
+return {
+  postgres = {
+    up = [[
+      DO $$
+      BEGIN
+      ALTER TABLE IF EXISTS ONLY "clustering_data_planes" ADD "labels" JSONB;
+      EXCEPTION WHEN DUPLICATE_COLUMN THEN
+        -- Do nothing, accept existing state
+      END;
+      $$;
+    ]]
+  }
+}

kong/db/migrations/core/init.lua

@@ -18,4 +18,5 @@ return {
   "018_310_to_320",
   "019_320_to_330",
   "020_330_to_340",
+  "021_340_to_350",
 }

kong/db/schema/entities/clustering_data_planes.lua

@@ -32,5 +32,11 @@ return {
                       description = "The status of the clustering data planes sync.",
                     }
     },
+    { labels = { type = "map",
+                 keys = { type = "string" },
+                 values = { type = "string" },
+                 description = "Custom key value pairs as meta-data for DPs.",
+               },
+    },
   },
 }

spec/01-unit/01-db/01-schema/13-cluster_status_spec.lua

@@ -53,4 +53,17 @@ describe("plugins", function()
     assert.is_true(ok)
     assert.is_nil(err)
   end)
+
+  it("accepts labels", function()
+    local ok, err = validate({
+      ip = "127.0.0.1",
+      hostname = "dp.example.com",
+      labels = {
+        deployment = "mycloud",
+        region = "us-east-1"
+      }
+    })
+    assert.is_true(ok)
+    assert.is_nil(err)
+  end)
 end)

spec/02-integration/03-db/13-cluster_status_spec.lua

@@ -44,5 +44,32 @@ for _, strategy in helpers.each_strategy() do
         assert.is_nil(err)
       end)
     end)
+
+    describe("labels", function()
+      it(":upsert()", function()
+        local p, err = db.clustering_data_planes:upsert({ id = "eb51145a-aaaa-bbbb-cccc-22087fb081db", },
+                                                 { config_hash = "a9a166c59873245db8f1a747ba9a80a7",
+                                                   hostname = "localhost",
+                                                   ip = "127.0.0.1",
+                                                   labels = {
+                                                     deployment = "mycloud",
+                                                     region = "us-east-1",
+                                                   }
+                                                 })
+
+        assert.is_truthy(p)
+        assert.is_nil(err)
+      end)
+
+      it(":update()", function()
+        -- this time update instead of insert
+        local p, err = db.clustering_data_planes:update({ id = "eb51145a-aaaa-bbbb-cccc-22087fb081db", },
+                                          { config_hash = "a9a166c59873245db8f1a747ba9a80a7",
+                                            labels = { deployment = "aws", region = "us-east-2" }
+                                          })
+        assert.is_truthy(p)
+        assert.is_nil(err)
+      end)
+    end)
   end) -- kong.db [strategy]
 end

spec/02-integration/09-hybrid_mode/01-sync_spec.lua

@@ -69,7 +69,7 @@ describe("CP/DP communication #" .. strategy, function()
             assert.near(14 * 86400, v.ttl, 3)
             assert.matches("^(%d+%.%d+)%.%d+", v.version)
             assert.equal(CLUSTERING_SYNC_STATUS.NORMAL, v.sync_status)
-
+            assert.equal(CLUSTERING_SYNC_STATUS.NORMAL, v.sync_status)
             return true
           end
         end
@@ -723,4 +723,65 @@ describe("CP/DP config sync #" .. strategy, function()
   end)
 end)
 
+describe("CP/DP labels #" .. strategy, function()
+
+  lazy_setup(function()
+    helpers.get_db_utils(strategy) -- runs migrations
+
+    assert(helpers.start_kong({
+      role = "control_plane",
+      cluster_cert = "spec/fixtures/kong_clustering.crt",
+      cluster_cert_key = "spec/fixtures/kong_clustering.key",
+      database = strategy,
+      db_update_frequency = 0.1,
+      cluster_listen = "127.0.0.1:9005",
+      nginx_conf = "spec/fixtures/custom_nginx.template",
+    }))
+
+    assert(helpers.start_kong({
+      role = "data_plane",
+      database = "off",
+      prefix = "servroot2",
+      cluster_cert = "spec/fixtures/kong_clustering.crt",
+      cluster_cert_key = "spec/fixtures/kong_clustering.key",
+      cluster_control_plane = "127.0.0.1:9005",
+      proxy_listen = "0.0.0.0:9002",
+      nginx_conf = "spec/fixtures/custom_nginx.template",
+      cluster_dp_labels="deployment:mycloud,region:us-east-1",
+    }))
+  end)
+
+  lazy_teardown(function()
+    helpers.stop_kong("servroot2")
+    helpers.stop_kong()
+  end)
+
+  describe("status API", function()
+    it("shows DP status", function()
+      helpers.wait_until(function()
+        local admin_client = helpers.admin_client()
+        finally(function()
+          admin_client:close()
+        end)
+
+        local res = assert(admin_client:get("/clustering/data-planes"))
+        local body = assert.res_status(200, res)
+        local json = cjson.decode(body)
+
+        for _, v in pairs(json.data) do
+          if v.ip == "127.0.0.1" then
+            assert.near(14 * 86400, v.ttl, 3)
+            assert.matches("^(%d+%.%d+)%.%d+", v.version)
+            assert.equal(CLUSTERING_SYNC_STATUS.NORMAL, v.sync_status)
+            assert.equal(CLUSTERING_SYNC_STATUS.NORMAL, v.sync_status)
+            assert.equal("mycloud", v.labels.deployment)
+            assert.equal("us-east-1", v.labels.region)
+            return true
+          end
+        end
+      end, 10)
+    end)
+  end)
+end)
+
 end

spec/05-migration/db/migrations/core/021_340_to_350_spec.lua

@@ -0,0 +1,7 @@
+local uh = require "spec/upgrade_helpers"
+
+describe("database migration", function()
+  uh.old_after_up("has created the expected new columns", function()
+    assert.table_has_column("clustering_data_planes", "labels", "jsonb")
+  end)
+end)