Split VSR matrix in 3 (#5883)

* Split VSR matrix in 3

* fix quotes

* simplify marker

.github/data/matrix-smoke-oss.json

@@ -0,0 +1,82 @@
+{
+  "images": [
+    {
+      "label": "ingresses 1/2",
+      "image": "debian",
+      "type": "oss",
+      "marker": "'ingresses and not annotations and not basic_auth and not hsts and not watch_namespace and not wildcard_tls'",
+      "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x"
+    },
+    {
+      "label": "ingresses 2/2",
+      "image": "debian",
+      "type": "oss",
+      "marker": "'annotations or basic_auth or hsts or watch_namespace or wildcard_tls'",
+      "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x"
+    },
+    {
+      "label": "VSR 1/3",
+      "image": "alpine",
+      "type": "oss",
+      "marker": "'vsr and not vsr_upstream and not vsr_grpc and not vsr_status and not vsr_canary and not vsr_routing and not vsr_api and not vsr_redirects and not vsr_rewrite and not vsr_canned and not vsr_basic'",
+      "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x"
+    },
+    {
+      "label": "VSR 2/3",
+      "image": "alpine",
+      "type": "oss",
+      "marker": "'vsr_basic or vsr_canned or vsr_rewrite or vsr_redirects or vsr_upstream'",
+      "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x"
+    },
+    {
+      "label": "VSR 3/3",
+      "image": "alpine",
+      "type": "oss",
+      "marker": "'vsr_api or vsr_routing or vsr_canary or vsr_status or vsr_grpc'",
+      "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x"
+    },
+    {
+      "label": "policies 1/2",
+      "image": "alpine",
+      "type": "oss",
+      "marker": "'policies and not policies_rl and not policies_ac and not policies_jwt and not policies_mtls'",
+      "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x"
+    },
+    {
+      "label": "policies 2/2",
+      "image": "alpine",
+      "type": "oss",
+      "marker": "'policies_rl or policies_ac or policies_jwt or policies_mtls'",
+      "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x"
+    },
+    {
+      "label": "VS 1/3",
+      "image": "debian",
+      "type": "oss",
+      "marker": "'vs and not vs_ipv6 and not vs_rewrite and not vs_responses and not vs_grpc and not vs_redirects and not vs_externalname and not vs_externaldns and not vs_certmanager'",
+      "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x"
+    },
+    {
+      "label": "VS 2/3",
+      "image": "debian",
+      "type": "oss",
+      "marker": "'vs_grpc or vs_redirects or vs_externalname or vs_externaldns'",
+      "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x"
+    },
+    {
+      "label": "VS 3/3",
+      "image": "debian",
+      "type": "oss",
+      "marker": "'vs_responses or vs_ipv6 or vs_rewrite or vs_certmanager'",
+      "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x"
+    },
+    {
+      "label": "TS",
+      "image": "ubi",
+      "type": "oss",
+      "marker": "ts",
+      "platforms": "linux/arm64, linux/amd64, linux/ppc64le, linux/s390x"
+    }
+  ],
+  "k8s": []
+}

.github/data/matrix-smoke-plus.json

@@ -0,0 +1,82 @@
+{
+  "images": [
+    {
+      "label": "VS 1/3",
+      "image": "debian-plus",
+      "type": "plus",
+      "marker": "'vs and not vs_ipv6 and not vs_rewrite and not vs_responses and not vs_grpc and not vs_redirects and not vs_externalname and not vs_externaldns and not vs_certmanager'",
+      "platforms": "linux/arm64, linux/amd64"
+    },
+    {
+      "label": "VS 2/3",
+      "image": "debian-plus",
+      "type": "plus",
+      "marker": "'vs_grpc or vs_redirects or vs_externalname or vs_externaldns'",
+      "platforms": "linux/arm64, linux/amd64"
+    },
+    {
+      "label": "VS 3/3",
+      "image": "debian-plus",
+      "type": "plus",
+      "marker": "'vs_responses or vs_ipv6 or vs_rewrite or vs_certmanager'",
+      "platforms": "linux/arm64, linux/amd64"
+    },
+    {
+      "label": "TS",
+      "image": "debian-plus",
+      "type": "plus",
+      "marker": "ts",
+      "platforms": "linux/arm64, linux/amd64"
+    },
+    {
+      "label": "ingresses 1/2",
+      "image": "alpine-plus",
+      "type": "plus",
+      "marker": "'ingresses and not annotations and not basic_auth and not hsts and not watch_namespace and not wildcard_tls'",
+      "platforms": "linux/arm64, linux/amd64"
+    },
+    {
+      "label": "ingresses 2/2",
+      "image": "alpine-plus",
+      "type": "plus",
+      "marker": "'annotations or basic_auth or hsts or watch_namespace or wildcard_tls'",
+      "platforms": "linux/arm64, linux/amd64"
+    },
+    {
+      "label": "VSR 1/3",
+      "image": "alpine-plus",
+      "type": "plus",
+      "marker": "'vsr and not vsr_upstream and not vsr_grpc and not vsr_status and not vsr_canary and not vsr_routing and not vsr_api and not vsr_redirects and not vsr_rewrite and not vsr_canned and not vsr_basic'",
+      "platforms": "linux/arm64, linux/amd64"
+    },
+    {
+      "label": "VSR 2/3",
+      "image": "alpine-plus",
+      "type": "plus",
+      "marker": "'vsr_basic or vsr_canned or vsr_rewrite or vsr_redirects or vsr_upstream'",
+      "platforms": "linux/arm64, linux/amd64"
+    },
+    {
+      "label": "VSR 3/3",
+      "image": "alpine-plus",
+      "type": "plus",
+      "marker": "'vsr_api or vsr_routing or vsr_canary or vsr_status or vsr_grpc'",
+      "platforms": "linux/arm64, linux/amd64"
+    },
+    {
+      "label": "policies 1/2",
+      "image": "ubi-plus",
+      "type": "plus",
+      "marker": "'policies and not policies_ac and not policies_jwt and not policies_mtls'",
+      "platforms": "linux/arm64, linux/amd64, linux/s390x"
+    },
+    {
+      "label": "policies 2/2",
+      "image": "ubi-plus",
+      "type": "plus",
+      "marker": "'policies_ac or policies_jwt or policies_mtls'",
+      "platforms": "linux/arm64, linux/amd64, linux/s390x"
+    }
+  ],
+  "k8s": []
+}

tests/suite/test_rewrites.py

@@ -112,6 +112,7 @@ def test_vs_rewrite(self, vs_rewrites_setup, path, args, cookies, expected):
         assert f"URI: {expected}\nRequest" in resp.text
 
     @pytest.mark.vsr
+    @pytest.mark.vsr_rewrite
     @pytest.mark.parametrize("path,args,cookies,expected", test_data)
     def test_vsr_rewrite(self, vsr_rewrites_setup, path, args, cookies, expected):
         """

tests/suite/test_v_s_route.py

@@ -49,6 +49,7 @@ def assert_locations_not_in_config(config, paths):
 
 @pytest.mark.smoke
 @pytest.mark.vsr
+@pytest.mark.vsr_basic
 @pytest.mark.parametrize(
     "crd_ingress_controller, v_s_route_setup",
     [({"type": "complete", "extra_args": [f"-enable-custom-resources"]}, {"example": "virtual-server-route"})],
@@ -267,6 +268,7 @@ def test_responses_and_events_in_flow(
 
 
 @pytest.mark.vsr
+@pytest.mark.vsr_basic
 @pytest.mark.parametrize(
     "crd_ingress_controller, v_s_route_setup",
     [({"type": "complete", "extra_args": [f"-enable-custom-resources"]}, {"example": "virtual-server-route"})],
@@ -369,6 +371,7 @@ def test_openapi_validation_flow(
 
 
 @pytest.mark.vsr
+@pytest.mark.vsr_basic
 @pytest.mark.flaky(max_runs=3)
 @pytest.mark.parametrize(
     "crd_ingress_controller, v_s_route_setup",

tests/suite/test_v_s_route_advanced_routing.py

@@ -107,6 +107,7 @@ def fin():
 
 
 @pytest.mark.vsr
+@pytest.mark.vsr_routing
 @pytest.mark.parametrize(
     "crd_ingress_controller, vsr_adv_routing_setup",
     [

tests/suite/test_v_s_route_api.py

@@ -8,6 +8,7 @@
 
 
 @pytest.mark.vsr
+@pytest.mark.vsr_api
 @pytest.mark.skip_for_nginx_oss
 @pytest.mark.parametrize(
     "crd_ingress_controller, v_s_route_setup",

tests/suite/test_v_s_route_canned_responses.py

@@ -15,6 +15,7 @@
 
 
 @pytest.mark.vsr
+@pytest.mark.vsr_canned
 @pytest.mark.parametrize(
     "crd_ingress_controller, v_s_route_setup",
     [

tests/suite/test_v_s_route_error_pages.py

@@ -14,6 +14,7 @@
 
 
 @pytest.mark.vsr
+@pytest.mark.vsr_error_pages
 @pytest.mark.parametrize(
     "crd_ingress_controller, v_s_route_setup",
     [

tests/suite/test_v_s_route_externalname.py

@@ -129,6 +129,7 @@ def fin():
 
 
 @pytest.mark.vsr
+@pytest.mark.vsr_external_name
 @pytest.mark.skip_for_nginx_oss
 @pytest.mark.parametrize(
     "crd_ingress_controller, vsr_externalname_setup",

tests/suite/test_v_s_route_focused_canary.py

@@ -116,6 +116,7 @@ def fin():
 
 @pytest.mark.flaky(max_runs=3)
 @pytest.mark.vsr
+@pytest.mark.vsr_canary
 @pytest.mark.parametrize(
     "crd_ingress_controller, vsr_canary_setup",
     [

tests/suite/test_v_s_route_grpc.py

@@ -68,6 +68,7 @@ def fin():
 
 
 @pytest.mark.vsr
+@pytest.mark.vsr_grpc
 @pytest.mark.smoke
 @pytest.mark.parametrize(
     "crd_ingress_controller, v_s_route_setup",

tests/suite/test_v_s_route_redirects.py

@@ -13,6 +13,7 @@
 
 
 @pytest.mark.vsr
+@pytest.mark.vsr_redirects
 @pytest.mark.parametrize(
     "crd_ingress_controller, v_s_route_setup",
     [

tests/suite/test_v_s_route_regexp_location.py

@@ -21,6 +21,7 @@
 
 
 @pytest.mark.vsr
+@pytest.mark.vsr_regexes
 @pytest.mark.parametrize(
     "crd_ingress_controller, v_s_route_setup",
     [
@@ -214,6 +215,7 @@ def vsr_regexp_setup(
 
 
 @pytest.mark.vsr
+@pytest.mark.vsr_regexes
 @pytest.mark.parametrize(
     "crd_ingress_controller, vsr_regexp_setup",
     [

tests/suite/test_v_s_route_split_traffic.py

@@ -40,6 +40,7 @@ def get_upstreams_of_splitting(file) -> []:
 
 
 @pytest.mark.vsr
+@pytest.mark.vsr_splits
 @pytest.mark.parametrize(
     "crd_ingress_controller, v_s_route_setup",
     [

tests/suite/test_v_s_route_status.py

@@ -11,6 +11,7 @@
 
 
 @pytest.mark.vsr
+@pytest.mark.vsr_status
 @pytest.mark.parametrize(
     "crd_ingress_controller, v_s_route_setup",
     [

tests/suite/test_v_s_route_upstream_options.py

@@ -26,6 +26,7 @@
     indirect=True,
 )
 @pytest.mark.vsr
+@pytest.mark.vsr_upstream
 class TestVSRouteUpstreamOptions:
     def test_nginx_config_upstreams_defaults(
         self, kube_apis, ingress_controller_prerequisites, crd_ingress_controller, v_s_route_setup, v_s_route_app_setup
@@ -393,6 +394,7 @@ def test_v_s_r_overrides_config_map(
 
 
 @pytest.mark.vsr
+@pytest.mark.vsr_upstream
 @pytest.mark.parametrize(
     "crd_ingress_controller, v_s_route_setup",
     [
@@ -544,6 +546,7 @@ def test_openapi_validation_flow(
 
 
 @pytest.mark.vsr
+@pytest.mark.vsr_upstream
 @pytest.mark.skip_for_nginx_oss
 @pytest.mark.parametrize(
     "crd_ingress_controller, v_s_route_setup",

tests/suite/test_v_s_route_upstream_tls.py

@@ -60,6 +60,7 @@ def fin():
 
 
 @pytest.mark.vsr
+@pytest.mark.vsr_upstream
 @pytest.mark.parametrize(
     "crd_ingress_controller, v_s_route_setup",
     [

tests/suite/test_v_s_route_weight_changes_dynamic_reload.py

@@ -94,6 +94,7 @@ def fin():
 
 
 @pytest.mark.vsr
+@pytest.mark.vsr_splits
 @pytest.mark.smok
 @pytest.mark.skip_for_nginx_oss
 @pytest.mark.parametrize(

tests/suite/test_v_s_route_weight_changes_dynamic_reload_many_splits.py

@@ -117,6 +117,7 @@ def fin():
 
 
 @pytest.mark.vsr
+@pytest.mark.vsr_splits
 @pytest.mark.smok
 @pytest.mark.skip_for_nginx_oss
 @pytest.mark.parametrize(

tests/suite/test_watch_secret_namespace.py

@@ -9,6 +9,7 @@
 
 
 @pytest.mark.vsr
+@pytest.mark.vsr_secrets
 @pytest.mark.parametrize(
     "crd_ingress_controller, v_s_route_setup",
     [
@@ -63,6 +64,7 @@ def test_responses(
 
 
 @pytest.mark.vsr
+@pytest.mark.vsr_secrets
 @pytest.mark.parametrize(
     "crd_ingress_controller, v_s_route_setup",
     [