[ZIv6F2QY] Allow Command Expansion APOC

LICENSES.txt

@@ -2815,9 +2815,11 @@ and/or involve the use of third party software.
 
 ------------------------------------------------------------------------------
 MIT
-  bcpkix-jdk15on-1.69.jar
-  bcprov-jdk15on-1.69.jar
-  bcutil-jdk15on-1.69.jar
+  bcpkix-jdk15on-1.68.jar
+  bcpkix-jdk18on-1.75.jar
+  bcprov-jdk15on-1.68.jar
+  bcprov-jdk18on-1.75.jar
+  bcutil-jdk18on-1.75.jar
   cassandra-1.17.6.jar
   checker-qual-3.33.0.jar
   couchbase-1.17.6.jar

NOTICE.txt

@@ -444,9 +444,11 @@ LGPL-2.1-or-later
   jna-5.9.0.jar
 
 MIT
-  bcpkix-jdk15on-1.69.jar
-  bcprov-jdk15on-1.69.jar
-  bcutil-jdk15on-1.69.jar
+  bcpkix-jdk15on-1.68.jar
+  bcpkix-jdk18on-1.75.jar
+  bcprov-jdk15on-1.68.jar
+  bcprov-jdk18on-1.75.jar
+  bcutil-jdk18on-1.75.jar
   cassandra-1.17.6.jar
   checker-qual-3.33.0.jar
   couchbase-1.17.6.jar

core/src/main/java/apoc/ApocConfig.java

@@ -20,6 +20,7 @@
 
 import apoc.export.util.ExportConfig;
 import apoc.util.SimpleRateLimiter;
+import com.google.api.client.util.Preconditions;
 import inet.ipaddr.IPAddressString;
 import org.apache.commons.configuration2.Configuration;
 import org.apache.commons.configuration2.PropertiesConfiguration;
@@ -41,6 +42,7 @@
 import org.neo4j.logging.NullLog;
 import org.neo4j.logging.internal.LogService;
 
+import java.io.File;
 import java.io.IOException;
 import java.lang.reflect.Field;
 import java.net.InetAddress;
@@ -56,8 +58,10 @@
 import java.util.stream.Stream;
 
 import static apoc.util.FileUtils.isFile;
+import static java.lang.String.format;
 import static org.neo4j.configuration.BootloaderSettings.lib_directory;
 import static org.neo4j.configuration.BootloaderSettings.run_directory;
+import static org.neo4j.configuration.Config.executeCommand;
 import static org.neo4j.configuration.GraphDatabaseInternalSettings.logical_logs_location;
 import static org.neo4j.configuration.GraphDatabaseSettings.SYSTEM_DATABASE_NAME;
 import static org.neo4j.configuration.GraphDatabaseSettings.data_directory;
@@ -131,6 +135,10 @@ public enum UuidFormatType { hex, base64 }
 
     private List<IPAddressString> blockedIpRanges = List.of();
 
+    private boolean expandCommands;
+
+    private Duration commandEvaluationTimeout;
+
     /**
      * keep track if this instance is already initialized so dependent class can wait if needed
      */
@@ -139,6 +147,11 @@ public enum UuidFormatType { hex, base64 }
     public ApocConfig(Config neo4jConfig, LogService log, GlobalProcedures globalProceduresRegistry, DatabaseManagementService databaseManagementService) {
         this.neo4jConfig = neo4jConfig;
         this.blockedIpRanges = neo4jConfig.get(ApocSettings.cypher_ip_blocklist);
+        this.commandEvaluationTimeout = neo4jConfig.get(GraphDatabaseInternalSettings.config_command_evaluation_timeout);
+        if (this.commandEvaluationTimeout == null) {
+            this.commandEvaluationTimeout = GraphDatabaseInternalSettings.config_command_evaluation_timeout.defaultValue();
+        }
+        this.expandCommands = neo4jConfig.expandCommands();
         this.log = log.getInternalLog(ApocConfig.class);
         this.databaseManagementService = databaseManagementService;
         theInstance = this;
@@ -160,6 +173,21 @@ public ApocConfig(Config neo4jConfig) {
         this.config = new PropertiesConfiguration();
     }
 
+    private String evaluateIfCommand(String settingName, String entry) {
+        if (Config.isCommand(entry)) {
+            Preconditions.checkArgument(
+                    expandCommands,
+                    format(
+                            "%s is a command, but config is not explicitly told to expand it. (Missing --expand-commands argument?)",
+                            entry));
+            String str = entry.trim();
+            String command = str.substring(2, str.length() - 1);
+            log.info("Executing external script to retrieve value of setting " + settingName);
+            return executeCommand(command, commandEvaluationTimeout);
+        }
+        return entry;
+    }
+
     public Configuration getConfig() {
         return config;
     }
@@ -171,6 +199,12 @@ public void init() throws Exception {
         String neo4jConfFolder = System.getenv().getOrDefault("NEO4J_CONF", determineNeo4jConfFolder());
         System.setProperty("NEO4J_CONF", neo4jConfFolder);
         log.info("system property NEO4J_CONF set to %s", neo4jConfFolder);
+        File apocConfFile = new File(neo4jConfFolder + "/apoc.conf");
+        // Command Expansion required check from Neo4j
+        if (apocConfFile.exists() && this.expandCommands) {
+            Config.Builder.validateFilePermissionForCommandExpansion(List.of(apocConfFile.toPath()));
+        }
+
         loadConfiguration();
         initialized = true;
     }
@@ -206,13 +240,18 @@ protected String determineNeo4jConfFolder() {
      */
     protected void loadConfiguration() {
         try {
-
             URL resource = getClass().getClassLoader().getResource("apoc-config.xml");
             log.info("loading apoc meta config from %s", resource.toString());
             CombinedConfigurationBuilder builder = new CombinedConfigurationBuilder()
                     .configure(new Parameters().fileBased().setURL(resource));
             config = builder.getConfiguration();
 
+            // Command Expansion if needed
+            config.getKeys().forEachRemaining(configKey -> config.setProperty(
+                    configKey,
+                    evaluateIfCommand(configKey, config.getProperty(configKey).toString())
+            ));
+
             // copy apoc settings from neo4j.conf for legacy support
             neo4jConfig.getDeclaredSettings().entrySet().stream()
                     .filter(e -> !config.containsKey(e.getKey()))

core/src/test/java/apoc/ApocConfigTest.java

@@ -18,6 +18,7 @@
  */
 package apoc;
 
+import org.assertj.core.api.Assertions;
 import org.junit.Before;
 import org.junit.Test;
 import org.neo4j.configuration.Config;
@@ -27,60 +28,179 @@
 import org.neo4j.logging.LogProvider;
 import org.neo4j.logging.internal.SimpleLogService;
 import org.neo4j.procedure.impl.GlobalProceduresRegistry;
+import org.testcontainers.shaded.com.google.common.collect.Sets;
 
 import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.nio.file.Files;
+import java.nio.file.attribute.PosixFilePermission;
 import java.util.Collections;
+import java.util.Set;
 
 import static apoc.ApocConfig.SUN_JAVA_COMMAND;
+import static java.nio.file.attribute.PosixFilePermission.GROUP_EXECUTE;
+import static java.nio.file.attribute.PosixFilePermission.GROUP_READ;
+import static java.nio.file.attribute.PosixFilePermission.GROUP_WRITE;
+import static java.nio.file.attribute.PosixFilePermission.OTHERS_EXECUTE;
+import static java.nio.file.attribute.PosixFilePermission.OTHERS_READ;
+import static java.nio.file.attribute.PosixFilePermission.OTHERS_WRITE;
+import static java.nio.file.attribute.PosixFilePermission.OWNER_EXECUTE;
+import static java.nio.file.attribute.PosixFilePermission.OWNER_READ;
+import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThrows;
 import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
 public class ApocConfigTest {
 
-    private ApocConfig cut;
+    private ApocConfig apocConfig;
+    private File apocConfigFile;
+    private static final Set<PosixFilePermission> permittedFilePermissionsForCommandExpansion =
+            Set.of(OWNER_READ, OWNER_WRITE, GROUP_READ);
+    private static final Set<PosixFilePermission> forbiddenFilePermissionsForCommandExpansion =
+            Set.of(OWNER_EXECUTE, GROUP_WRITE, GROUP_EXECUTE, OTHERS_READ, OTHERS_WRITE, OTHERS_EXECUTE);
 
     @Before
-    public void setup() {
+    public void setup() throws Exception {
         LogProvider logProvider = new AssertableLogProvider();
 
         Config neo4jConfig = mock(Config.class);
         when(neo4jConfig.getDeclaredSettings()).thenReturn(Collections.emptyMap());
         when(neo4jConfig.get(any())).thenReturn(null);
         when(neo4jConfig.get(GraphDatabaseSettings.allow_file_urls)).thenReturn(false);
+        when(neo4jConfig.expandCommands()).thenReturn(true);
+
+        apocConfigFile = new File(getClass().getClassLoader().getResource("apoc.conf").toURI());
+        Files.setPosixFilePermissions(apocConfigFile.toPath(), permittedFilePermissionsForCommandExpansion);
 
         GlobalProceduresRegistry registry = mock(GlobalProceduresRegistry.class);
         DatabaseManagementService databaseManagementService = mock(DatabaseManagementService.class);
-        cut = new ApocConfig(neo4jConfig, new SimpleLogService(logProvider), registry, databaseManagementService);
+        apocConfig = new ApocConfig(neo4jConfig, new SimpleLogService(logProvider), registry, databaseManagementService);
+    }
+
+
+    private void setApocConfigFilePermissions(Set<PosixFilePermission> forbidden) throws Exception {
+        Files.setPosixFilePermissions(
+                apocConfigFile.toPath(),
+                Sets.union(permittedFilePermissionsForCommandExpansion, forbidden)
+        );
+    }
+
+    private void setApocConfigSystemProperty() {
+        System.setProperty(SUN_JAVA_COMMAND, "com.neo4j.server.enterprise.CommercialEntryPoint --home-dir=/home/stefan/neo4j-enterprise-4.0.0-alpha09mr02 --config-dir=" +  apocConfigFile.getParent());
     }
 
     @Test
     public void testDetermineNeo4jConfFolderDefault() {
         System.setProperty(SUN_JAVA_COMMAND, "");
-        assertEquals(".", cut.determineNeo4jConfFolder());
+        assertEquals(".", apocConfig.determineNeo4jConfFolder());
     }
 
     @Test
     public void testDetermineNeo4jConfFolder() {
         System.setProperty(SUN_JAVA_COMMAND, "com.neo4j.server.enterprise.CommercialEntryPoint --home-dir=/home/stefan/neo4j-enterprise-4.0.0-alpha09mr02 --config-dir=/home/stefan/neo4j-enterprise-4.0.0-alpha09mr02/conf");
 
-        assertEquals("/home/stefan/neo4j-enterprise-4.0.0-alpha09mr02/conf", cut.determineNeo4jConfFolder());
+        assertEquals("/home/stefan/neo4j-enterprise-4.0.0-alpha09mr02/conf", apocConfig.determineNeo4jConfFolder());
     }
 
     @Test
     public void testApocConfFileBeingLoaded() throws Exception {
         String confDir = new File(getClass().getClassLoader().getResource("apoc.conf").toURI()).getParent();
         System.setProperty(SUN_JAVA_COMMAND, "com.neo4j.server.enterprise.CommercialEntryPoint --home-dir=/home/stefan/neo4j-enterprise-4.0.0-alpha09mr02 --config-dir=" + confDir);
-        cut.init();
+        apocConfig.init();
 
-        assertEquals("bar", cut.getConfig().getString("foo"));
+        assertEquals("bar", apocConfig.getConfig().getString("foo"));
     }
 
     @Test
     public void testDetermineNeo4jConfFolderWithWhitespaces() {
         System.setProperty(SUN_JAVA_COMMAND, "com.neo4j.server.enterprise.CommercialEntryPoint --config-dir=/home/stefan/neo4j enterprise-4.0.0-alpha09mr02/conf --home-dir=/home/stefan/neo4j enterprise-4.0.0-alpha09mr02");
 
-        assertEquals("/home/stefan/neo4j enterprise-4.0.0-alpha09mr02/conf", cut.determineNeo4jConfFolder());
+        assertEquals("/home/stefan/neo4j enterprise-4.0.0-alpha09mr02/conf", apocConfig.determineNeo4jConfFolder());
+    }
+
+    @Test
+    public void testApocConfWithExpandCommands() throws Exception {
+        String validExpandLine = "command.expansion=$(echo \"expanded value\")";
+        addLineToApocConfig(validExpandLine);
+        setApocConfigSystemProperty();
+
+        apocConfig.init();
+
+        assertEquals("expanded value", apocConfig.getConfig().getString("command.expansion"));
+        removeLineFromApocConfig(validExpandLine);
+    }
+
+    @Test
+    public void testApocConfWithInvalidExpandCommands() throws Exception {
+        String invalidExpandLine = "command.expansion.3=$(fakeCommand 3 + 3)";
+        addLineToApocConfig(invalidExpandLine);
+        setApocConfigSystemProperty();
+
+        RuntimeException e = assertThrows(RuntimeException.class, apocConfig::init);
+        String expectedMessage = "java.io.IOException: Cannot run program \"fakeCommand\": error=2, No such file or directory";
+        Assertions.assertThat(e.getMessage()).contains(expectedMessage);
+
+        removeLineFromApocConfig(invalidExpandLine);
+    }
+
+    @Test
+    public void testApocConfWithWrongFilePermissions() throws Exception {
+        for (PosixFilePermission filePermission : forbiddenFilePermissionsForCommandExpansion) {
+            setApocConfigFilePermissions(Set.of(filePermission));
+            setApocConfigSystemProperty();
+
+            RuntimeException e = assertThrows(RuntimeException.class, apocConfig::init);
+            String expectedMessage = "does not have the correct file permissions to evaluate commands.";
+            Assertions.assertThat(e.getMessage()).contains(expectedMessage);
+        }
+        // Set back to permitted after test
+        setApocConfigFilePermissions(permittedFilePermissionsForCommandExpansion);
     }
+
+    @Test
+    public void testApocConfWithoutExpandCommands() throws Exception {
+        LogProvider logProvider = new AssertableLogProvider();
+
+        Config neo4jConfig = mock(Config.class);
+        when(neo4jConfig.getDeclaredSettings()).thenReturn(Collections.emptyMap());
+        when(neo4jConfig.get( any())).thenReturn(null);
+        when(neo4jConfig.expandCommands()).thenReturn(false);
+
+        GlobalProceduresRegistry registry = mock(GlobalProceduresRegistry.class);
+        DatabaseManagementService databaseManagementService = mock(DatabaseManagementService.class);
+        ApocConfig apocConfig = new ApocConfig(neo4jConfig, new SimpleLogService(logProvider), registry, databaseManagementService);
+
+        String validExpandLine = "command.expansion=$(echo \"expanded value\")";
+        addLineToApocConfig(validExpandLine);
+        setApocConfigSystemProperty();
+
+        RuntimeException e = assertThrows(RuntimeException.class, apocConfig::init);
+        String expectedMessage = "$(echo \"expanded value\") is a command, but config is not explicitly told to expand it. (Missing --expand-commands argument?)";
+        Assertions.assertThat(e.getMessage()).contains(expectedMessage);
+
+        removeLineFromApocConfig(validExpandLine);
+    }
+
+    private void removeLineFromApocConfig(String lineContent) throws IOException {
+        File temp = new File("_temp_");
+        PrintWriter out = new PrintWriter(new FileWriter(temp));
+        Files.lines(apocConfigFile.toPath())
+                .filter(line -> !line.contains(lineContent))
+                .forEach(out::println);
+        out.flush();
+        out.close();
+        temp.renameTo(apocConfigFile);
+    }
+
+    private void addLineToApocConfig(String line) throws IOException {
+        FileWriter fw = new FileWriter(apocConfigFile,true);
+        fw.write(line);
+        fw.close();
+    }
+
 }