diff --git a/LICENSES.txt b/LICENSES.txt index e05ad03951..18f0ec9dbf 100644 --- a/LICENSES.txt +++ b/LICENSES.txt @@ -2815,9 +2815,11 @@ and/or involve the use of third party software. ------------------------------------------------------------------------------ MIT - bcpkix-jdk15on-1.69.jar - bcprov-jdk15on-1.69.jar - bcutil-jdk15on-1.69.jar + bcpkix-jdk15on-1.68.jar + bcpkix-jdk18on-1.75.jar + bcprov-jdk15on-1.68.jar + bcprov-jdk18on-1.75.jar + bcutil-jdk18on-1.75.jar cassandra-1.17.6.jar checker-qual-3.33.0.jar couchbase-1.17.6.jar diff --git a/NOTICE.txt b/NOTICE.txt index 36be185344..c818dcf2f0 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -444,9 +444,11 @@ LGPL-2.1-or-later jna-5.9.0.jar MIT - bcpkix-jdk15on-1.69.jar - bcprov-jdk15on-1.69.jar - bcutil-jdk15on-1.69.jar + bcpkix-jdk15on-1.68.jar + bcpkix-jdk18on-1.75.jar + bcprov-jdk15on-1.68.jar + bcprov-jdk18on-1.75.jar + bcutil-jdk18on-1.75.jar cassandra-1.17.6.jar checker-qual-3.33.0.jar couchbase-1.17.6.jar diff --git a/core/src/main/java/apoc/ApocConfig.java b/core/src/main/java/apoc/ApocConfig.java index 1e877ee12e..ad9ad95da4 100644 --- a/core/src/main/java/apoc/ApocConfig.java +++ b/core/src/main/java/apoc/ApocConfig.java @@ -20,6 +20,7 @@ import apoc.export.util.ExportConfig; import apoc.util.SimpleRateLimiter; +import com.google.api.client.util.Preconditions; import inet.ipaddr.IPAddressString; import org.apache.commons.configuration2.Configuration; import org.apache.commons.configuration2.PropertiesConfiguration; @@ -41,6 +42,7 @@ import org.neo4j.logging.NullLog; import org.neo4j.logging.internal.LogService; +import java.io.File; import java.io.IOException; import java.lang.reflect.Field; import java.net.InetAddress; @@ -56,8 +58,10 @@ import java.util.stream.Stream; import static apoc.util.FileUtils.isFile; +import static java.lang.String.format; import static org.neo4j.configuration.BootloaderSettings.lib_directory; import static org.neo4j.configuration.BootloaderSettings.run_directory; +import static org.neo4j.configuration.Config.executeCommand; import static org.neo4j.configuration.GraphDatabaseInternalSettings.logical_logs_location; import static org.neo4j.configuration.GraphDatabaseSettings.SYSTEM_DATABASE_NAME; import static org.neo4j.configuration.GraphDatabaseSettings.data_directory; @@ -131,6 +135,10 @@ public enum UuidFormatType { hex, base64 } private List blockedIpRanges = List.of(); + private boolean expandCommands; + + private Duration commandEvaluationTimeout; + /** * keep track if this instance is already initialized so dependent class can wait if needed */ @@ -139,6 +147,11 @@ public enum UuidFormatType { hex, base64 } public ApocConfig(Config neo4jConfig, LogService log, GlobalProcedures globalProceduresRegistry, DatabaseManagementService databaseManagementService) { this.neo4jConfig = neo4jConfig; this.blockedIpRanges = neo4jConfig.get(ApocSettings.cypher_ip_blocklist); + this.commandEvaluationTimeout = neo4jConfig.get(GraphDatabaseInternalSettings.config_command_evaluation_timeout); + if (this.commandEvaluationTimeout == null) { + this.commandEvaluationTimeout = GraphDatabaseInternalSettings.config_command_evaluation_timeout.defaultValue(); + } + this.expandCommands = neo4jConfig.expandCommands(); this.log = log.getInternalLog(ApocConfig.class); this.databaseManagementService = databaseManagementService; theInstance = this; @@ -160,6 +173,21 @@ public ApocConfig(Config neo4jConfig) { this.config = new PropertiesConfiguration(); } + private String evaluateIfCommand(String settingName, String entry) { + if (Config.isCommand(entry)) { + Preconditions.checkArgument( + expandCommands, + format( + "%s is a command, but config is not explicitly told to expand it. (Missing --expand-commands argument?)", + entry)); + String str = entry.trim(); + String command = str.substring(2, str.length() - 1); + log.info("Executing external script to retrieve value of setting " + settingName); + return executeCommand(command, commandEvaluationTimeout); + } + return entry; + } + public Configuration getConfig() { return config; } @@ -171,6 +199,12 @@ public void init() throws Exception { String neo4jConfFolder = System.getenv().getOrDefault("NEO4J_CONF", determineNeo4jConfFolder()); System.setProperty("NEO4J_CONF", neo4jConfFolder); log.info("system property NEO4J_CONF set to %s", neo4jConfFolder); + File apocConfFile = new File(neo4jConfFolder + "/apoc.conf"); + // Command Expansion required check from Neo4j + if (apocConfFile.exists() && this.expandCommands) { + Config.Builder.validateFilePermissionForCommandExpansion(List.of(apocConfFile.toPath())); + } + loadConfiguration(); initialized = true; } @@ -206,13 +240,18 @@ protected String determineNeo4jConfFolder() { */ protected void loadConfiguration() { try { - URL resource = getClass().getClassLoader().getResource("apoc-config.xml"); log.info("loading apoc meta config from %s", resource.toString()); CombinedConfigurationBuilder builder = new CombinedConfigurationBuilder() .configure(new Parameters().fileBased().setURL(resource)); config = builder.getConfiguration(); + // Command Expansion if needed + config.getKeys().forEachRemaining(configKey -> config.setProperty( + configKey, + evaluateIfCommand(configKey, config.getProperty(configKey).toString()) + )); + // copy apoc settings from neo4j.conf for legacy support neo4jConfig.getDeclaredSettings().entrySet().stream() .filter(e -> !config.containsKey(e.getKey())) diff --git a/core/src/test/java/apoc/ApocConfigTest.java b/core/src/test/java/apoc/ApocConfigTest.java index 935bb23787..d3cbc52b09 100644 --- a/core/src/test/java/apoc/ApocConfigTest.java +++ b/core/src/test/java/apoc/ApocConfigTest.java @@ -18,6 +18,7 @@ */ package apoc; +import org.assertj.core.api.Assertions; import org.junit.Before; import org.junit.Test; import org.neo4j.configuration.Config; @@ -27,60 +28,179 @@ import org.neo4j.logging.LogProvider; import org.neo4j.logging.internal.SimpleLogService; import org.neo4j.procedure.impl.GlobalProceduresRegistry; +import org.testcontainers.shaded.com.google.common.collect.Sets; import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintWriter; +import java.nio.file.Files; +import java.nio.file.attribute.PosixFilePermission; import java.util.Collections; +import java.util.Set; import static apoc.ApocConfig.SUN_JAVA_COMMAND; +import static java.nio.file.attribute.PosixFilePermission.GROUP_EXECUTE; +import static java.nio.file.attribute.PosixFilePermission.GROUP_READ; +import static java.nio.file.attribute.PosixFilePermission.GROUP_WRITE; +import static java.nio.file.attribute.PosixFilePermission.OTHERS_EXECUTE; +import static java.nio.file.attribute.PosixFilePermission.OTHERS_READ; +import static java.nio.file.attribute.PosixFilePermission.OTHERS_WRITE; +import static java.nio.file.attribute.PosixFilePermission.OWNER_EXECUTE; +import static java.nio.file.attribute.PosixFilePermission.OWNER_READ; +import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class ApocConfigTest { - private ApocConfig cut; + private ApocConfig apocConfig; + private File apocConfigFile; + private static final Set permittedFilePermissionsForCommandExpansion = + Set.of(OWNER_READ, OWNER_WRITE, GROUP_READ); + private static final Set forbiddenFilePermissionsForCommandExpansion = + Set.of(OWNER_EXECUTE, GROUP_WRITE, GROUP_EXECUTE, OTHERS_READ, OTHERS_WRITE, OTHERS_EXECUTE); @Before - public void setup() { + public void setup() throws Exception { LogProvider logProvider = new AssertableLogProvider(); Config neo4jConfig = mock(Config.class); when(neo4jConfig.getDeclaredSettings()).thenReturn(Collections.emptyMap()); when(neo4jConfig.get(any())).thenReturn(null); when(neo4jConfig.get(GraphDatabaseSettings.allow_file_urls)).thenReturn(false); + when(neo4jConfig.expandCommands()).thenReturn(true); + + apocConfigFile = new File(getClass().getClassLoader().getResource("apoc.conf").toURI()); + Files.setPosixFilePermissions(apocConfigFile.toPath(), permittedFilePermissionsForCommandExpansion); GlobalProceduresRegistry registry = mock(GlobalProceduresRegistry.class); DatabaseManagementService databaseManagementService = mock(DatabaseManagementService.class); - cut = new ApocConfig(neo4jConfig, new SimpleLogService(logProvider), registry, databaseManagementService); + apocConfig = new ApocConfig(neo4jConfig, new SimpleLogService(logProvider), registry, databaseManagementService); + } + + + private void setApocConfigFilePermissions(Set forbidden) throws Exception { + Files.setPosixFilePermissions( + apocConfigFile.toPath(), + Sets.union(permittedFilePermissionsForCommandExpansion, forbidden) + ); + } + + private void setApocConfigSystemProperty() { + System.setProperty(SUN_JAVA_COMMAND, "com.neo4j.server.enterprise.CommercialEntryPoint --home-dir=/home/stefan/neo4j-enterprise-4.0.0-alpha09mr02 --config-dir=" + apocConfigFile.getParent()); } @Test public void testDetermineNeo4jConfFolderDefault() { System.setProperty(SUN_JAVA_COMMAND, ""); - assertEquals(".", cut.determineNeo4jConfFolder()); + assertEquals(".", apocConfig.determineNeo4jConfFolder()); } @Test public void testDetermineNeo4jConfFolder() { System.setProperty(SUN_JAVA_COMMAND, "com.neo4j.server.enterprise.CommercialEntryPoint --home-dir=/home/stefan/neo4j-enterprise-4.0.0-alpha09mr02 --config-dir=/home/stefan/neo4j-enterprise-4.0.0-alpha09mr02/conf"); - assertEquals("/home/stefan/neo4j-enterprise-4.0.0-alpha09mr02/conf", cut.determineNeo4jConfFolder()); + assertEquals("/home/stefan/neo4j-enterprise-4.0.0-alpha09mr02/conf", apocConfig.determineNeo4jConfFolder()); } @Test public void testApocConfFileBeingLoaded() throws Exception { String confDir = new File(getClass().getClassLoader().getResource("apoc.conf").toURI()).getParent(); System.setProperty(SUN_JAVA_COMMAND, "com.neo4j.server.enterprise.CommercialEntryPoint --home-dir=/home/stefan/neo4j-enterprise-4.0.0-alpha09mr02 --config-dir=" + confDir); - cut.init(); + apocConfig.init(); - assertEquals("bar", cut.getConfig().getString("foo")); + assertEquals("bar", apocConfig.getConfig().getString("foo")); } @Test public void testDetermineNeo4jConfFolderWithWhitespaces() { System.setProperty(SUN_JAVA_COMMAND, "com.neo4j.server.enterprise.CommercialEntryPoint --config-dir=/home/stefan/neo4j enterprise-4.0.0-alpha09mr02/conf --home-dir=/home/stefan/neo4j enterprise-4.0.0-alpha09mr02"); - assertEquals("/home/stefan/neo4j enterprise-4.0.0-alpha09mr02/conf", cut.determineNeo4jConfFolder()); + assertEquals("/home/stefan/neo4j enterprise-4.0.0-alpha09mr02/conf", apocConfig.determineNeo4jConfFolder()); + } + + @Test + public void testApocConfWithExpandCommands() throws Exception { + String validExpandLine = "command.expansion=$(echo \"expanded value\")"; + addLineToApocConfig(validExpandLine); + setApocConfigSystemProperty(); + + apocConfig.init(); + + assertEquals("expanded value", apocConfig.getConfig().getString("command.expansion")); + removeLineFromApocConfig(validExpandLine); + } + + @Test + public void testApocConfWithInvalidExpandCommands() throws Exception { + String invalidExpandLine = "command.expansion.3=$(fakeCommand 3 + 3)"; + addLineToApocConfig(invalidExpandLine); + setApocConfigSystemProperty(); + + RuntimeException e = assertThrows(RuntimeException.class, apocConfig::init); + String expectedMessage = "java.io.IOException: Cannot run program \"fakeCommand\": error=2, No such file or directory"; + Assertions.assertThat(e.getMessage()).contains(expectedMessage); + + removeLineFromApocConfig(invalidExpandLine); + } + + @Test + public void testApocConfWithWrongFilePermissions() throws Exception { + for (PosixFilePermission filePermission : forbiddenFilePermissionsForCommandExpansion) { + setApocConfigFilePermissions(Set.of(filePermission)); + setApocConfigSystemProperty(); + + RuntimeException e = assertThrows(RuntimeException.class, apocConfig::init); + String expectedMessage = "does not have the correct file permissions to evaluate commands."; + Assertions.assertThat(e.getMessage()).contains(expectedMessage); + } + // Set back to permitted after test + setApocConfigFilePermissions(permittedFilePermissionsForCommandExpansion); } + + @Test + public void testApocConfWithoutExpandCommands() throws Exception { + LogProvider logProvider = new AssertableLogProvider(); + + Config neo4jConfig = mock(Config.class); + when(neo4jConfig.getDeclaredSettings()).thenReturn(Collections.emptyMap()); + when(neo4jConfig.get( any())).thenReturn(null); + when(neo4jConfig.expandCommands()).thenReturn(false); + + GlobalProceduresRegistry registry = mock(GlobalProceduresRegistry.class); + DatabaseManagementService databaseManagementService = mock(DatabaseManagementService.class); + ApocConfig apocConfig = new ApocConfig(neo4jConfig, new SimpleLogService(logProvider), registry, databaseManagementService); + + String validExpandLine = "command.expansion=$(echo \"expanded value\")"; + addLineToApocConfig(validExpandLine); + setApocConfigSystemProperty(); + + RuntimeException e = assertThrows(RuntimeException.class, apocConfig::init); + String expectedMessage = "$(echo \"expanded value\") is a command, but config is not explicitly told to expand it. (Missing --expand-commands argument?)"; + Assertions.assertThat(e.getMessage()).contains(expectedMessage); + + removeLineFromApocConfig(validExpandLine); + } + + private void removeLineFromApocConfig(String lineContent) throws IOException { + File temp = new File("_temp_"); + PrintWriter out = new PrintWriter(new FileWriter(temp)); + Files.lines(apocConfigFile.toPath()) + .filter(line -> !line.contains(lineContent)) + .forEach(out::println); + out.flush(); + out.close(); + temp.renameTo(apocConfigFile); + } + + private void addLineToApocConfig(String line) throws IOException { + FileWriter fw = new FileWriter(apocConfigFile,true); + fw.write(line); + fw.close(); + } + }