diff --git a/servo/connectors/kubernetes.py b/servo/connectors/kubernetes.py index 373d0de5b..b2ee5fd95 100644 --- a/servo/connectors/kubernetes.py +++ b/servo/connectors/kubernetes.py @@ -2997,6 +2997,9 @@ async def create( f'no container named "{container_config.name}" exists in the Pod (found {names})' ) + if container_config.static_environment_variables: + raise NotImplementedError("Configurable environment variables are not currently supported under Deployment optimization (saturation mode)") + name = container_config.alias or ( f"{deployment.name}/{container.name}" if container else deployment.name ) @@ -3420,6 +3423,22 @@ async def _configure_tuning_pod_template_spec(self) -> None: container = Container(container_obj, None) servo.logger.debug(f"Initialized new tuning container from Pod spec template: {container.name}") + if self.container_config.static_environment_variables: + if container.obj.env is None: + container.obj.env = [] + + # Filter out vars with the same name as the ones we are setting + container.obj.env = list(filter( + lambda e: e.name not in self.container_config.static_environment_variables, + container.obj.env + )) + + env_list = [ + kubernetes_asyncio.client.V1EnvVar(name=k, value=v) + for k, v in self.container_config.static_environment_variables.items() + ] + container.obj.env.extend(env_list) + if self.tuning_container: servo.logger.debug(f"Copying resource requirements from existing tuning pod container '{self.tuning_pod.name}/{self.tuning_container.name}'") resource_requirements = self.tuning_container.resources @@ -4067,7 +4086,8 @@ class ContainerConfiguration(servo.BaseConfiguration): command: Optional[str] # TODO: create model... cpu: CPU memory: Memory - env: Optional[List[str]] # TODO: create model... + env: Optional[List[str]] # (adjustable environment variables) TODO: create model... + static_environment_variables: Optional[Dict[str, str]] diff --git a/servo/connectors/opsani_dev.py b/servo/connectors/opsani_dev.py index 427be6c5a..c072295e9 100644 --- a/servo/connectors/opsani_dev.py +++ b/servo/connectors/opsani_dev.py @@ -57,6 +57,7 @@ class OpsaniDevConfiguration(servo.BaseConfiguration): port: Optional[Union[pydantic.StrictInt, str]] = None cpu: CPU memory: Memory + static_environment_variables: Optional[Dict[str, str]] prometheus_base_url: str = PROMETHEUS_SIDECAR_BASE_URL envoy_sidecar_image: str = ENVOY_SIDECAR_IMAGE_TAG timeout: servo.Duration = "5m" @@ -109,6 +110,7 @@ def generate_kubernetes_config( alias="main", cpu=self.cpu, memory=self.memory, + static_environment_variables=self.static_environment_variables, ) ], ) diff --git a/tests/connectors/kubernetes_test.py b/tests/connectors/kubernetes_test.py index 4cb9ba8a2..c4dcebf16 100644 --- a/tests/connectors/kubernetes_test.py +++ b/tests/connectors/kubernetes_test.py @@ -45,7 +45,7 @@ ) from servo.errors import AdjustmentFailedError, AdjustmentRejectedError import servo.runner -from servo.types import Adjustment +from servo.types import Adjustment, Component, Description, Replicas from tests.helpers import * @@ -1164,12 +1164,46 @@ async def test_read_pod(self, config, kube) -> None: ## # Canary Tests - # async def test_create_canary(self, tuning_config, namespace: str) -> None: - # connector = KubernetesConnector(config=tuning_config) - # dep = await Deployment.read("fiber-http", namespace) - # debug(dep) - # description = await connector.startup() - # debug(description) + async def test_create_tuning( + self, + tuning_config: KubernetesConfiguration, + kube: kubetest.client.TestClient + ) -> None: + # verify existing env vars are overriden by config var with same name + main_dep = kube.get_deployments()["fiber-http"] + main_dep.obj.spec.template.spec.containers[0].env = [kubernetes.client.models.V1EnvVar(name="FOO", value="BAZ")] + main_dep.api_client.patch_namespaced_deployment(main_dep.name, main_dep.namespace, main_dep.obj) + tuning_config.deployments[0].containers[0].static_environment_variables = { "FOO": "BAR" } + + connector = KubernetesConnector(config=tuning_config) + description = await connector.describe() + + assert description == Description(components=[ + Component( + name='fiber-http/fiber-http', + settings=[ + CPU(name='cpu', type='range', pinned=True, value="125m", min="125m", max="875m", step="125m", request="125m", limit="125m", get=['request', 'limit'], set=['request', 'limit']), + Memory(name='mem', type='range', pinned=True, value=134217728, min=134217728, max=805306368, step=33554432, request=134217728, limit=134217728, get=['request', 'limit'], set=['request', 'limit']), + Replicas(name='replicas', type='range', pinned=True, value=1, min=0, max=99999, step=1) + ] + ), + Component( + name='fiber-http/fiber-http-tuning', + settings=[ + CPU(name='cpu', type='range', pinned=False, value="125m", min="125m", max="875m", step="125m", request="125m", limit="125m", get=['request', 'limit'], set=['request', 'limit']), + Memory(name='mem', type='range', pinned=False, value=134217728, min=134217728, max=805306368, step=33554432, request=134217728, limit=134217728, get=['request', 'limit'], set=['request', 'limit']), + Replicas(name='replicas', type='range', pinned=True, value=1, min=0, max=1, step=1) + ] + ) + ]) + + tuning_pod = kube.get_pods()["fiber-http-tuning"] + assert tuning_pod.obj.metadata.annotations["opsani.com/opsani_tuning_for"] == "fiber-http/fiber-http-tuning" + assert tuning_pod.obj.metadata.labels["opsani_role"] == "tuning" + target_container = next(filter(lambda c: c.name == "fiber-http" , tuning_pod.obj.spec.containers)) + assert target_container.resources.requests == {'cpu': '125m', 'memory': '128Mi'} + assert target_container.resources.limits == {'cpu': '125m', 'memory': '128Mi'} + assert target_container.env == [kubernetes.client.models.V1EnvVar(name="FOO", value="BAR")] async def test_adjust_tuning_insufficient_mem( self, @@ -2384,10 +2418,40 @@ def _rollout_tuning_config(self, tuning_config: KubernetesConfiguration) -> Kube ## # Canary Tests - async def test_create_rollout_tuning(self, _rollout_tuning_config: KubernetesConfiguration, namespace: str) -> None: + async def test_create_rollout_tuning( + self, _rollout_tuning_config: KubernetesConfiguration, kube: kubetest.client.TestClient, namespace: str + ) -> None: + _rollout_tuning_config.rollouts[0].containers[0].static_environment_variables = { "FOO": "BAR" } connector = KubernetesConnector(config=_rollout_tuning_config) rol = await Rollout.read("fiber-http", namespace) - await connector.describe() + description = await connector.describe() + + assert description == Description(components=[ + Component( + name='fiber-http/fiber-http', + settings=[ + CPU(name='cpu', type='range', pinned=True, value="125m", min="125m", max="875m", step="125m", request="125m", limit="125m", get=['request', 'limit'], set=['request', 'limit']), + Memory(name='mem', type='range', pinned=True, value=134217728, min=134217728, max=805306368, step=33554432, request=134217728, limit=134217728, get=['request', 'limit'], set=['request', 'limit']), + Replicas(name='replicas', type='range', pinned=True, value=1, min=0, max=99999, step=1) + ] + ), + Component( + name='fiber-http/fiber-http-tuning', + settings=[ + CPU(name='cpu', type='range', pinned=False, value="125m", min="125m", max="875m", step="125m", request="125m", limit="125m", get=['request', 'limit'], set=['request', 'limit']), + Memory(name='mem', type='range', pinned=False, value=134217728, min=134217728, max=805306368, step=33554432, request=134217728, limit=134217728, get=['request', 'limit'], set=['request', 'limit']), + Replicas(name='replicas', type='range', pinned=True, value=1, min=0, max=1, step=1) + ] + ) + ]) + + tuning_pod = kube.get_pods()["fiber-http-tuning"] + assert tuning_pod.obj.metadata.annotations["opsani.com/opsani_tuning_for"] == "fiber-http/fiber-http-tuning" + assert tuning_pod.obj.metadata.labels["opsani_role"] == "tuning" + target_container = next(filter(lambda c: c.name == "fiber-http" , tuning_pod.obj.spec.containers)) + assert target_container.resources.requests == {'cpu': '125m', 'memory': '128Mi'} + assert target_container.resources.limits == {'cpu': '125m', 'memory': '128Mi'} + assert target_container.env == [kubernetes.client.models.V1EnvVar(name="FOO", value="BAR")] # verify tuning pod is registered as service endpoint service = await servo.connectors.kubernetes.Service.read("fiber-http", namespace) diff --git a/tests/connectors/opsani_dev_test.py b/tests/connectors/opsani_dev_test.py index af605e2b5..f7fee820d 100644 --- a/tests/connectors/opsani_dev_test.py +++ b/tests/connectors/opsani_dev_test.py @@ -71,7 +71,10 @@ def rollout_checks(rollout_config: servo.connectors.opsani_dev.OpsaniDevConfigur class TestConfig: def test_generate(self) -> None: config = servo.connectors.opsani_dev.OpsaniDevConfiguration.generate() - assert list(config.dict().keys()) == ['description', 'namespace', 'deployment', 'rollout', 'container', 'service', 'port', 'cpu', 'memory', 'prometheus_base_url', 'envoy_sidecar_image', 'timeout', 'settlement'] + assert list(config.dict().keys()) == [ + 'description', 'namespace', 'deployment', 'rollout', 'container', 'service','port', 'cpu', 'memory', + 'static_environment_variables', 'prometheus_base_url', 'envoy_sidecar_image', 'timeout', 'settlement' + ] def test_generate_yaml(self) -> None: config = servo.connectors.opsani_dev.OpsaniDevConfiguration.generate() @@ -92,6 +95,26 @@ def test_assign_optimizer(self) -> None: config = servo.connectors.opsani_dev.OpsaniDevConfiguration.generate() config.__optimizer__ = None + def test_generate_kubernetes_config(self) -> None: + opsani_dev_config = servo.connectors.opsani_dev.OpsaniDevConfiguration( + namespace="test", + deployment="fiber-http", + container="fiber-http", + service="fiber-http", + cpu=servo.connectors.kubernetes.CPU(min="125m", max="4000m", step="125m"), + memory=servo.connectors.kubernetes.Memory(min="128 MiB", max="4.0 GiB", step="128 MiB"), + static_environment_variables={"FOO": "BAR", "BAZ": 1}, + __optimizer__=servo.configuration.Optimizer(id="test.com/foo", token="12345") + ) + kubernetes_config = opsani_dev_config.generate_kubernetes_config() + assert kubernetes_config.namespace == "test" + assert kubernetes_config.deployments[0].namespace == "test" + assert kubernetes_config.deployments[0].name == "fiber-http" + assert kubernetes_config.deployments[0].containers[0].name == "fiber-http" + assert kubernetes_config.deployments[0].containers[0].cpu == servo.connectors.kubernetes.CPU(min="125m", max="4000m", step="125m") + assert kubernetes_config.deployments[0].containers[0].memory == servo.connectors.kubernetes.Memory(min="128 MiB", max="4.0 GiB", step="128 MiB") + assert kubernetes_config.deployments[0].containers[0].static_environment_variables == {"FOO": "BAR", "BAZ": "1"} + def test_generate_rollout_config(self) -> None: rollout_config = servo.connectors.opsani_dev.OpsaniDevConfiguration( namespace="test",