Add systemd integ tests to run with docker

qa/systemd-test/build.gradle

@@ -0,0 +1,26 @@
+import org.opensearch.gradle.Architecture
+import org.opensearch.gradle.VersionProperties
+import org.opensearch.gradle.testfixtures.TestFixturesPlugin
+
+apply plugin: 'opensearch.standalone-rest-test'
+apply plugin: 'opensearch.test.fixtures'
+
+testFixtures.useFixture()
+
+dockerCompose {
+  useComposeFiles = ['docker-compose.yml']
+}
+
+
+tasks.register("integTest", Test) {
+  outputs.doNotCacheIf('Build cache is disabled for Docker tests') { true }
+  maxParallelForks = '1'
+  include '**/*IT.class'
+}
+
+tasks.named("check").configure { dependsOn "integTest" }
+
+tasks.named("integTest").configure {
+  dependsOn "composeUp"
+  finalizedBy "composeDown"
+}

qa/systemd-test/docker-compose.yml

@@ -0,0 +1,64 @@
+services:
+  # self-contained systemd example: run 'docker-compose up' to see it
+  centos:
+    image: opensearch-systemd-test
+    container_name: opensearch-systemd-test-container
+    build:
+      dockerfile_inline: |
+        FROM centos:8
+        RUN sed -i 's|mirrorlist=|#mirrorlist=|g' /etc/yum.repos.d/*.repo && \
+        sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/*.repo
+
+        # install systemd
+        RUN dnf -y install systemd && dnf clean all
+        # in practice, you'd COPY in the RPM you want to test right here
+        RUN dnf -y install https://artifacts.opensearch.org/releases/bundle/opensearch/2.18.0/opensearch-2.18.0-linux-x64.rpm && dnf clean all
+        # add a test-user
+        RUN useradd -ms /bin/bash testuser
+        # no colors
+        ENV SYSTEMD_COLORS=0
+        # no escapes
+        ENV SYSTEMD_URLIFY=0
+        # explicitly specify docker virtualization
+        ENV container=docker
+        # for debugging systemd issues in container, you want this, but it is very loud!
+        # ENV SYSTEMD_LOG_LEVEL=debug
+        # plumb journald logs to stdout
+        COPY <<EOF /etc/systemd/journald.conf
+        [Journal]
+        ForwardToConsole=yes
+        EOF
+        # start systemd as PID 1
+        CMD ["usr/sbin/init"]
+        # enable opensearch service
+        RUN systemctl enable opensearch
+        # shutdown systemd properly
+        STOPSIGNAL SIGRTMIN+3
+        # disable security plugin, as i don't configure SSL (but could be done with openssl or whatever right here)
+        RUN echo "plugins.security.disabled: true" >> /etc/opensearch/opensearch.yml
+        RUN echo "network.host: 0.0.0.0" >> /etc/opensearch/opensearch.yml
+        RUN echo "discovery.type: single-node" >> /etc/opensearch/opensearch.yml
+    # provide /dev/console for journal logs to go to stdout
+    tty: true
+    # capabilities to allow systemd to sandbox
+    cap_add:
+      # https://systemd.io/CONTAINER_INTERFACE/#what-you-shouldnt-do bullet 1
+      - SYS_ADMIN
+      # https://systemd.io/CONTAINER_INTERFACE/#what-you-shouldnt-do bullet 2
+      - MKNOD
+    # evil, but best you can do on docker? podman is better here.
+    cgroup: host
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup
+    # tmpfs mounts for systemd
+    tmpfs:
+      - /run
+      - /run/lock
+    # health check for opensearch
+    ports:
+      - "9200:9200"
+      - "9300:9300"
+    privileged: true
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9200/_cluster/health"]
+      start_period: 15s

qa/systemd-test/src/test/java/org/opensearch/systemdinteg/SystemdIT.java

@@ -0,0 +1,235 @@
+/*
+* SPDX-License-Identifier: Apache-2.0
+*
+* The OpenSearch Contributors require contributions made to
+* this file be licensed under the Apache-2.0 license or a
+* compatible open source license.
+*/
+
+/*
+* Licensed to Elasticsearch under one or more contributor
+* license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright
+* ownership. Elasticsearch licenses this file to you under
+* the Apache License, Version 2.0 (the "License"); you may
+* not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*    http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+/*
+* Modifications Copyright OpenSearch Contributors. See
+* GitHub history for details.
+*/
+
+package org.opensearch.systemdinteg;
+
+import org.apache.hc.core5.http.HttpHeaders;
+import org.apache.hc.core5.http.HttpHost;
+import org.apache.hc.core5.http.HttpStatus;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.BufferedReader;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+
+public class SystemdIT {
+    private static final String OPENSEARCH_URL = "http://localhost:9200"; // OpenSearch URL (port 9200)
+    private static String containerId;
+    private static String opensearchPid;
+    private static final String CONTAINER_NAME = "opensearch-systemd-test-container";
+
+    @BeforeClass
+    public static void setup() throws IOException, InterruptedException {
+        containerId = getContainerId();
+
+        String status = executeCommand("docker exec " + containerId + " systemctl status opensearch", "Failed to check OpenSearch status");
+
+        opensearchPid = getOpenSearchPid();
+
+        if (opensearchPid.isEmpty()) {
+            throw new RuntimeException("Failed to find OpenSearch process ID");
+        }
+    }
+
+    private static String getContainerId() throws IOException, InterruptedException {
+        return executeCommand("docker ps -qf name=" + CONTAINER_NAME, "OpenSearch container '" + CONTAINER_NAME + "' is not running");
+    }
+
+    private static String getOpenSearchPid() throws IOException, InterruptedException {
+        String command = "docker exec " + containerId + " systemctl show --property=MainPID opensearch";
+        String output = executeCommand(command, "Failed to get OpenSearch PID");
+        return output.replace("MainPID=", "").trim();
+    }
+
+    private boolean checkPathExists(String path) throws IOException, InterruptedException {
+        String command = String.format("docker exec %s test -e %s && echo true || echo false", containerId, path);
+        return Boolean.parseBoolean(executeCommand(command, "Failed to check path existence"));
+    }
+
+    private boolean checkPathReadable(String path) throws IOException, InterruptedException {
+        String command = String.format("docker exec %s su opensearch -s /bin/sh -c 'test -r %s && echo true || echo false'", containerId, path);
+        return Boolean.parseBoolean(executeCommand(command, "Failed to check read permission"));
+    }
+
+    private boolean checkPathWritable(String path) throws IOException, InterruptedException {
+        String command = String.format("docker exec %s su opensearch -s /bin/sh -c 'test -w %s && echo true || echo false'", containerId, path);
+        return Boolean.parseBoolean(executeCommand(command, "Failed to check write permission"));
+    }
+
+    private String getPathOwnership(String path) throws IOException, InterruptedException {
+        String command = String.format("docker exec %s stat -c '%%U:%%G' %s", containerId, path);
+        return executeCommand(command, "Failed to get path ownership");
+    }
+
+    private static String executeCommand(String command, String errorMessage) throws IOException, InterruptedException {
+        Process process = Runtime.getRuntime().exec(new String[]{"bash", "-c", command});
+        try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) {
+            StringBuilder output = new StringBuilder();
+            String line;
+            while ((line = reader.readLine()) != null) {
+                output.append(line).append("\n");
+            }
+            if (process.waitFor() != 0) {
+                throw new RuntimeException(errorMessage);
+            }
+            return output.toString().trim();
+        }
+    }
+
+    @Test
+    public void testClusterHealth() throws IOException {
+        HttpURLConnection healthCheck = (HttpURLConnection) new URL(OPENSEARCH_URL + "/_cluster/health").openConnection();
+        healthCheck.setRequestMethod("GET");
+        int healthResponseCode = healthCheck.getResponseCode();
+        assertTrue(healthResponseCode == HttpURLConnection.HTTP_OK);
+    }
+
+    @Test
+    public void testMaxProcesses() throws IOException, InterruptedException {
+        String limits = executeCommand("docker exec " + containerId + " cat /proc/" + opensearchPid + "/limits", "Failed to read process limits");
+        assertTrue("Max processes limit should be 4096 or unlimited", 
+                limits.contains("Max processes             4096                 4096") ||
+                limits.contains("Max processes             unlimited            unlimited"));
+    }
+
+    @Test
+    public void testFileDescriptorLimit() throws IOException, InterruptedException {
+        String limits = executeCommand("docker exec " + containerId + " cat /proc/" + opensearchPid + "/limits", "Failed to read process limits");
+        assertTrue("File descriptor limit should be at least 65535", 
+                limits.contains("Max open files            65535                65535") ||
+                limits.contains("Max open files            unlimited            unlimited"));
+    }
+
+    @Test
+    public void testSystemCallFilter() throws IOException, InterruptedException {
+        // Check if Seccomp is enabled
+        String seccomp = executeCommand("docker exec " + containerId + " grep Seccomp /proc/" + opensearchPid + "/status", "Failed to read Seccomp status");
+        assertFalse("Seccomp should be enabled", seccomp.contains("0"));
+
+        // Test specific system calls that should be blocked
+        String rebootResult = executeCommand("docker exec " + containerId + " su opensearch -c 'kill -s SIGHUP 1' 2>&1 || echo 'Operation not permitted'", "Failed to test reboot system call");
+        assertTrue("Reboot system call should be blocked", rebootResult.contains("Operation not permitted"));
+
+        String swapResult = executeCommand("docker exec " + containerId + " su opensearch -c 'swapon -a' 2>&1 || echo 'Operation not permitted'", "Failed to test swap system call");
+        assertTrue("Swap system call should be blocked", swapResult.contains("Operation not permitted"));
+    }
+
+    @Test
+    public void testReadOnlyPaths() throws IOException, InterruptedException {
+        String[] readOnlyPaths = {
+            "/etc/os-release", "/usr/lib/os-release", "/etc/system-release",
+            "/proc/self/mountinfo", "/proc/diskstats",
+            "/proc/self/cgroup", "/sys/fs/cgroup/cpu", "/sys/fs/cgroup/cpu/-",
+            "/sys/fs/cgroup/cpuacct", "/sys/fs/cgroup/cpuacct/-",
+            "/sys/fs/cgroup/memory", "/sys/fs/cgroup/memory/-"
+        };
+
+        for (String path : readOnlyPaths) {
+            if (checkPathExists(path)) {
+                assertTrue("Path should be readable: " + path, checkPathReadable(path));
+                assertFalse("Path should not be writable: " + path, checkPathWritable(path));
+            }
+        }
+    }
+
+    @Test
+    public void testReadWritePaths() throws IOException, InterruptedException {
+        String[] readWritePaths = {"/var/log/opensearch", "/var/lib/opensearch"};
+        for (String path : readWritePaths) {
+            assertTrue("Path should exist: " + path, checkPathExists(path));
+            assertTrue("Path should be readable: " + path, checkPathReadable(path));
+            assertTrue("Path should be writable: " + path, checkPathWritable(path));
+            assertEquals("Path should be owned by opensearch:opensearch", "opensearch:opensearch", getPathOwnership(path));
+        }
+    }
+
+    @Test
+    public void testProcessExit() throws IOException, InterruptedException {
+
+        String scriptContent = "#!/bin/sh\n" +
+                            "if [ $# -ne 1 ]; then\n" +
+                            "    echo \"Usage: $0 <PID>\"\n" +
+                            "    exit 1\n" +
+                            "fi\n" +
+                            "if kill -15 $1 2>/dev/null; then\n" +
+                            "    echo \"SIGTERM signal sent to process $1\"\n" +
+                            "else\n" +
+                            "    echo \"Failed to send SIGTERM to process $1\"\n" +
+                            "fi\n" +
+                            "sleep 2\n" +
+                            "if kill -0 $1 2>/dev/null; then\n" +
+                            "    echo \"Process $1 is still running\"\n" +
+                            "else\n" +
+                            "    echo \"Process $1 has terminated\"\n" +
+                            "fi";
+
+        String[] command = {
+            "docker",
+            "exec",
+            "-u", "testuser",
+            containerId,
+            "sh",
+            "-c",
+            "echo '" + scriptContent.replace("'", "'\"'\"'") + "' > /tmp/terminate.sh && chmod +x /tmp/terminate.sh && /tmp/terminate.sh " + opensearchPid
+        };
+
+        ProcessBuilder processBuilder = new ProcessBuilder(command);
+        Process process = processBuilder.start();
+
+        // Wait a moment for any potential termination to take effect
+        Thread.sleep(2000);
+
+        // Check if the OpenSearch process is still running
+        String processCheck = executeCommand(
+            "docker exec " + containerId + " kill -0 " + opensearchPid + " 2>/dev/null && echo 'Running' || echo 'Not running'",
+            "Failed to check process status"
+        );
+
+        // Verify the OpenSearch service status
+        String serviceStatus = executeCommand(
+            "docker exec " + containerId + " systemctl is-active opensearch",
+            "Failed to check OpenSearch service status"
+        );
+
+        assertTrue("OpenSearch process should still be running", processCheck.contains("Running"));
+        assertEquals("OpenSearch service should be active", "active", serviceStatus.trim());
+    }
+
+}