Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix Kubernetes test scenario #1083

Merged
merged 2 commits into from
Apr 21, 2024
Merged

Fix Kubernetes test scenario #1083

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d0a4676
Fix Kubernetes test scenario
gtroitsk Feb 11, 2024
fa8c848
Add Kubernetes test for PostgreSQL
gtroitsk Mar 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions examples/database-postgresql/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,11 @@
<artifactId>quarkus-test-openshift</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.quarkus.qe</groupId>
<artifactId>quarkus-test-kubernetes</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
Expand Down
28 changes: 28 additions & 0 deletions ...se-postgresql/src/test/java/io/quarkus/qe/database/postgresql/KubernetesPostgresqlIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
package io.quarkus.qe.database.postgresql;

import io.quarkus.test.bootstrap.PostgresqlService;
import io.quarkus.test.bootstrap.RestService;
import io.quarkus.test.scenarios.KubernetesScenario;
import io.quarkus.test.services.Container;
import io.quarkus.test.services.QuarkusApplication;

@KubernetesScenario
public class KubernetesPostgresqlIT extends AbstractSqlDatabaseIT {

private static final int POSTGRESQL_PORT = 5432;

@Container(image = "${postgresql.image}", port = POSTGRESQL_PORT, expectedLog = "is ready")
static PostgresqlService database = new PostgresqlService()
.withProperty("PGDATA", "/tmp/psql");

@QuarkusApplication
static RestService app = new RestService()
.withProperty("quarkus.datasource.username", database.getUser())
.withProperty("quarkus.datasource.password", database.getPassword())
.withProperty("quarkus.datasource.jdbc.url", database::getJdbcUrl);

@Override
protected RestService getApp() {
return app;
}
}
3 changes: 1 addition & 2 deletions ...ples/pingpong/src/test/java/io/quarkus/qe/KubernetesUsingExtensionPingPongResourceIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,7 @@
public class KubernetesUsingExtensionPingPongResourceIT {

@QuarkusApplication
static final RestService pingpong = new RestService()
.withProperty("quarkus.kubernetes.service-type", "LoadBalancer");
static final RestService pingpong = new RestService();

@Test
public void shouldPingPongIsUpAndRunning() {
Expand Down
17 changes: 17 additions & 0 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -501,6 +501,23 @@
<include.tests>**/Kubernetes*IT.java</include.tests>
<exclude.kubernetes.tests>no</exclude.kubernetes.tests>
</properties>
<build>
<plugins>
<plugin>
<artifactId>maven-failsafe-plugin</artifactId>
<executions>
<execution>
<configuration>
<systemPropertyVariables>
<!-- always set 'Kubernetes' property as that's how detect Kubernetes tests inside FW -->
<kubernetes>true</kubernetes>
</systemPropertyVariables>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>validate-format</id>
Expand Down
9 changes: 8 additions & 1 deletion quarkus-test-kubernetes/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,14 @@
</dependency>
<dependency>
<groupId>io.fabric8</groupId>
<artifactId>openshift-client</artifactId>
<artifactId>kubernetes-client</artifactId>
<!-- Excludes artifact from CVE-2023-0833 -->
<exclusions>
<exclusion>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.rest-assured</groupId>
Expand Down
124 changes: 55 additions & 69 deletions quarkus-test-kubernetes/src/main/java/io/quarkus/test/bootstrap/inject/KubectlClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,9 @@
import static org.junit.jupiter.api.Assertions.fail;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collections;
Expand All @@ -24,6 +25,7 @@
import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.function.UnaryOperator;
import java.util.regex.Pattern;
import java.util.stream.Stream;
Expand All @@ -40,11 +42,13 @@
import io.fabric8.kubernetes.api.model.VolumeMount;
import io.fabric8.kubernetes.api.model.VolumeMountBuilder;
import io.fabric8.kubernetes.api.model.apps.Deployment;
import io.fabric8.kubernetes.client.Config;
import io.fabric8.kubernetes.client.ConfigBuilder;
import io.fabric8.kubernetes.client.KubernetesClientBuilder;
import io.fabric8.kubernetes.client.dsl.NamespaceListVisitFromServerGetDeleteRecreateWaitApplicable;
import io.fabric8.kubernetes.client.dsl.NonDeletingOperation;
import io.fabric8.kubernetes.client.impl.KubernetesClientImpl;
import io.fabric8.kubernetes.client.utils.Serialization;
import io.fabric8.openshift.client.DefaultOpenShiftClient;
import io.fabric8.openshift.client.NamespacedOpenShiftClient;
import io.fabric8.openshift.client.OpenShiftConfig;
import io.fabric8.openshift.client.OpenShiftConfigBuilder;
import io.quarkus.test.bootstrap.Service;
import io.quarkus.test.configuration.PropertyLookup;
import io.quarkus.test.logging.Log;
Expand All @@ -58,29 +62,37 @@ public final class KubectlClient {
public static final String LABEL_SCENARIO_ID = "scenarioId";
public static final PropertyLookup ENABLED_EPHEMERAL_NAMESPACES = new PropertyLookup(
"ts.kubernetes.ephemeral.namespaces.enabled", Boolean.TRUE.toString());

private static final String RESOURCE_MNT_FOLDER = "/resource";
private static final int NAMESPACE_NAME_SIZE = 10;
private static final int NAMESPACE_CREATION_RETRIES = 5;

private static final int DEPLOYMENT_CREATION_TIMEOUT = 30;

private static final String KUBECTL = "kubectl";
private static final int HTTP_PORT_DEFAULT = 80;

private final String currentNamespace;
private final DefaultOpenShiftClient masterClient;
private final NamespacedOpenShiftClient client;
private final KubernetesClientImpl client;
private final String scenarioId;

private KubectlClient(String scenarioUniqueName) {
this.scenarioId = scenarioUniqueName;
String activeNamespace = new DefaultOpenShiftClient().getNamespace();
currentNamespace = ENABLED_EPHEMERAL_NAMESPACES.getAsBoolean() ? createNamespace() : activeNamespace;
OpenShiftConfig config = new OpenShiftConfigBuilder().withTrustCerts(true).withNamespace(currentNamespace).build();
masterClient = new DefaultOpenShiftClient(config);
client = masterClient.inNamespace(currentNamespace);
if (ENABLED_EPHEMERAL_NAMESPACES.getAsBoolean()) {
currentNamespace = createNamespace();
Config config = new ConfigBuilder().withTrustCerts(true).withNamespace(currentNamespace).build();
client = createClient(config);
} else {
Config config = new ConfigBuilder().withTrustCerts(true).build();
client = createClient(config);
currentNamespace = client.getNamespace();
}
setCurrentSessionNamespace(currentNamespace);
}

private static KubernetesClientImpl createClient(Config config) {
return new KubernetesClientBuilder().withConfig(config).build()
.adapt(KubernetesClientImpl.class);
}

public static KubectlClient create(String scenarioName) {
return new KubectlClient(scenarioName);
}
Expand Down Expand Up @@ -121,7 +133,8 @@ public void applyServicePropertiesUsingDeploymentConfig(Service service) {
envVar -> container.getEnv().add(new EnvVar(envVar.getKey(), envVar.getValue(), null)));
});

client.apps().deployments().createOrReplace(deployment);
client.apps().deployments().withTimeout(DEPLOYMENT_CREATION_TIMEOUT, TimeUnit.SECONDS).delete();
client.apps().deployments().resource(deployment).create();
}

/**
Expand Down Expand Up @@ -152,7 +165,7 @@ public void applyServiceProperties(Service service, String file, UnaryOperator<S
public void expose(Service service, Integer port) {
try {
new Command(KUBECTL, "expose", "deployment", service.getName(), "--port=" + port, "--name=" + service.getName(),
"--type=LoadBalancer", "-n", currentNamespace).runAndWait();
"--type=NodePort", "-n", currentNamespace).runAndWait();
} catch (Exception e) {
fail("Service failed to be exposed. Caused by " + e.getMessage());
}
Expand Down Expand Up @@ -214,36 +227,16 @@ public Map<String, String> logs(Service service) {
}

/**
* Resolve the url by the service.
*
* @param service
* @return
* Get node host IP.
*/
public String host(Service service) {
String serviceName = service.getName();
io.fabric8.kubernetes.api.model.Service serviceModel = client.services().withName(serviceName).get();
if (serviceModel == null
|| serviceModel.getStatus() == null
|| serviceModel.getStatus().getLoadBalancer() == null
|| serviceModel.getStatus().getLoadBalancer().getIngress() == null) {
printServiceInfo(service);
fail("Service " + serviceName + " not found");
}

// IP detection rules:
// 1.- Try Ingress IP
// 2.- Try Ingress Hostname
Optional<String> ip = serviceModel.getStatus().getLoadBalancer().getIngress().stream()
.map(ingress -> StringUtils.defaultIfBlank(ingress.getIp(), ingress.getHostname()))
.filter(StringUtils::isNotEmpty)
.findFirst();

if (ip.isEmpty()) {
printServiceInfo(service);
fail("Service " + serviceName + " host not found");
public String host() {
String nodeURL = client.network().getConfiguration().getMasterUrl();
try {
URI uri = new URI(nodeURL);
return uri.getHost();
} catch (URISyntaxException e) {
throw new IllegalStateException(e);
}

return ip.get();
}

/**
Expand All @@ -260,7 +253,7 @@ public int port(Service service) {
}

return serviceModel.getSpec().getPorts().stream()
.map(ServicePort::getPort)
.map(ServicePort::getNodePort)
.filter(Objects::nonNull)
.findFirst()
.orElse(HTTP_PORT_DEFAULT);
Expand All @@ -276,7 +269,7 @@ public void deleteNamespace() {
} catch (Exception e) {
fail("Project failed to be deleted. Caused by " + e.getMessage());
} finally {
masterClient.close();
client.close();
}
} else {
deleteResourcesByLabel(LABEL_SCENARIO_ID, getScenarioId());
Expand All @@ -297,7 +290,7 @@ private void deleteResourcesByLabel(String labelName, String labelValue) {
} catch (Exception e) {
fail("Project failed to be deleted. Caused by " + e.getMessage());
} finally {
masterClient.close();
client.close();
}
}

Expand All @@ -306,13 +299,14 @@ private boolean isPodRunning(Pod pod) {
}

private List<HasMetadata> loadYaml(String template) {
return client.load(new ByteArrayInputStream(template.getBytes())).get();
NamespaceListVisitFromServerGetDeleteRecreateWaitApplicable<HasMetadata> load = client
.load(new ByteArrayInputStream(template.getBytes()));
return load.items();
}

private String enrichTemplate(Service service, String template, Map<String, String> extraTemplateProperties) {
List<HasMetadata> objs = loadYaml(template);
for (HasMetadata obj : objs) {
// set namespace
List<HasMetadata> objects = loadYaml(template);
for (HasMetadata obj : objects) {
obj.getMetadata().setNamespace(namespace());
Map<String, String> objMetadataLabels = Optional.ofNullable(obj.getMetadata().getLabels())
.orElse(new HashMap<>());
Expand All @@ -321,23 +315,23 @@ private String enrichTemplate(Service service, String template, Map<String, Stri
obj.getMetadata().setLabels(objMetadataLabels);

if (obj instanceof Deployment) {
Deployment d = (Deployment) obj;
Deployment deployment = (Deployment) obj;

// set deployment name
d.getMetadata().setName(service.getName());
deployment.getMetadata().setName(service.getName());

// set metadata to template
d.getSpec().getTemplate().getMetadata().setNamespace(namespace());
deployment.getSpec().getTemplate().getMetadata().setNamespace(namespace());

// add label for logs
Map<String, String> templateMetadataLabels = d.getSpec().getTemplate().getMetadata().getLabels();
Map<String, String> templateMetadataLabels = deployment.getSpec().getTemplate().getMetadata().getLabels();
templateMetadataLabels.put(LABEL_TO_WATCH_FOR_LOGS, service.getName());
templateMetadataLabels.put(LABEL_SCENARIO_ID, getScenarioId());

// add env var properties
Map<String, String> enrichProperties = enrichProperties(service.getProperties(), d);
Map<String, String> enrichProperties = enrichProperties(service.getProperties(), deployment);
enrichProperties.putAll(extraTemplateProperties);
d.getSpec().getTemplate().getSpec().getContainers()
deployment.getSpec().getTemplate().getSpec().getContainers()
.forEach(container -> enrichProperties.entrySet().forEach(property -> {
String key = property.getKey();
EnvVar envVar = getEnvVarByKey(key, container);
Expand All @@ -351,16 +345,8 @@ private String enrichTemplate(Service service, String template, Map<String, Stri
}

KubernetesList list = new KubernetesList();
list.setItems(objs);
try {
ByteArrayOutputStream os = new ByteArrayOutputStream();
Serialization.yamlMapper().writeValue(os, list);
template = new String(os.toByteArray());
} catch (IOException e) {
fail("Failed adding properties into template. Caused by " + e.getMessage());
}

return template;
list.setItems(objects);
return Serialization.asYaml(list);
}

private EnvVar getEnvVarByKey(String key, Container container) {
Expand Down Expand Up @@ -461,10 +447,10 @@ private void createOrUpdateConfigMap(String configMapName, String key, String va
});
} else {
// create new one
client.configMaps().createOrReplace(new ConfigMapBuilder()
client.configMaps().resource(new ConfigMapBuilder()
.withNewMetadata().withName(configMapName).endMetadata()
.addToData(key, value)
.build());
.build()).createOr(NonDeletingOperation::update);
}
}

Expand Down
8 changes: 5 additions & 3 deletions ...src/main/java/io/quarkus/test/services/containers/KubernetesContainerManagedResource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@

import static java.util.regex.Pattern.quote;

import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;

import org.apache.commons.lang3.StringUtils;

Expand Down Expand Up @@ -73,7 +75,7 @@ public URILike getURI(Protocol protocol) {
}

return createURI("http",
client.host(model.getContext().getOwner()),
client.host(),
client.port(model.getContext().getOwner()));
}

Expand All @@ -100,11 +102,11 @@ private String replaceDeploymentContent(String content) {
// replace it by the service owner name
content = content.replaceAll(quote(customServiceName), model.getContext().getName());
}

String args = Arrays.stream(model.getCommand()).map(cmd -> "\"" + cmd + "\"").collect(Collectors.joining(", "));
return content.replaceAll(quote("${IMAGE}"), model.getImage())
.replaceAll(quote("${SERVICE_NAME}"), model.getContext().getName())
.replaceAll(quote("${INTERNAL_PORT}"), "" + model.getPort())
.replaceAll(quote("${ARGS}"), String.join(" ", model.getCommand()));
.replaceAll(quote("${ARGS}"), args);
}

private boolean useInternalServiceAsUrl() {
Expand Down
1 change: 0 additions & 1 deletion ...s/test/services/quarkus/ContainerRegistryKubernetesQuarkusApplicationManagedResource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,6 @@ private String replaceDeploymentContent(String content) {
return content
.replaceAll(quote("${IMAGE}"), image)
.replaceAll(quote("${SERVICE_NAME}"), model.getContext().getName())
.replaceAll(quote("${ARTIFACT}"), model.getArtifact().getFileName().toString())
.replaceAll(quote("${INTERNAL_PORT}"),
model.getContext().getOwner().getProperty(QUARKUS_HTTP_PORT_PROPERTY, "" + HTTP_PORT_DEFAULT));
}
Expand Down
Loading