From 835582369b958a5db96f1fad3acebc69777a1f52 Mon Sep 17 00:00:00 2001 From: Sebastian Peter Date: Mon, 18 Nov 2024 21:14:11 +0100 Subject: [PATCH 1/3] Refactoring ResultFileHierarchy Signed-off-by: Sebastian Peter --- .../event/listener/ResultEventListener.scala | 2 +- .../logback/LogbackConfiguration.scala | 10 +- .../scala/edu/ie3/simona/main/RunSimona.scala | 4 +- .../ie3/simona/sim/setup/SetupHelper.scala | 9 +- .../ie3/simona/sim/setup/SimonaSetup.scala | 4 +- .../sim/setup/SimonaStandaloneSetup.scala | 3 +- .../ie3/simona/util/ResultFileHierarchy.scala | 283 +++++++++--------- .../listener/ResultEventListenerSpec.scala | 85 +++--- .../integration/RunSimonaStandaloneIT.scala | 12 +- .../io/file/ResultFileHierarchySpec.scala | 64 +--- .../edu/ie3/simona/sim/SimonaSimSpec.scala | 3 +- .../simona/sim/setup/SimonaSetupSpec.scala | 3 +- 12 files changed, 230 insertions(+), 252 deletions(-) diff --git a/src/main/scala/edu/ie3/simona/event/listener/ResultEventListener.scala b/src/main/scala/edu/ie3/simona/event/listener/ResultEventListener.scala index 8e7dd12abd..d9c910ef1a 100644 --- a/src/main/scala/edu/ie3/simona/event/listener/ResultEventListener.scala +++ b/src/main/scala/edu/ie3/simona/event/listener/ResultEventListener.scala @@ -88,7 +88,7 @@ object ResultEventListener extends Transformer3wResultSupport { filePathFuture.map { fileName => val finalFileName = - fileName match { + fileName.toString match { case name if name.endsWith(".csv.gz") && enableCompression => name.replace(".gz", "") case name if name.endsWith(".csv") => name diff --git a/src/main/scala/edu/ie3/simona/logging/logback/LogbackConfiguration.scala b/src/main/scala/edu/ie3/simona/logging/logback/LogbackConfiguration.scala index 3d236a1f41..9f95df1c2c 100644 --- a/src/main/scala/edu/ie3/simona/logging/logback/LogbackConfiguration.scala +++ b/src/main/scala/edu/ie3/simona/logging/logback/LogbackConfiguration.scala @@ -16,16 +16,16 @@ import ch.qos.logback.core.filter.Filter import com.typesafe.scalalogging.LazyLogging import org.slf4j.LoggerFactory -import java.io.File +import java.nio.file.Path import scala.jdk.CollectionConverters._ object LogbackConfiguration extends LazyLogging { - def default(logPath: String): Unit = { + def default(logPath: Path): Unit = { LoggerFactory.getILoggerFactory match { case loggerContext: LoggerContext => val rootLogger = loggerContext.getLogger("root") - val log = logPath.concat(File.separator).concat("simona.log") + val log = logPath.resolve("simona.log") // stop all appenders rootLogger.iteratorForAppenders().asScala.foreach(_.stop()) /* Identify the filters of existing rolling file appender */ @@ -58,7 +58,7 @@ object LogbackConfiguration extends LazyLogging { } private def fileAppender( - logPath: String, + logPath: Path, appenderName: String, maybeFilterList: Option[Seq[Filter[ILoggingEvent]]], loggerContext: LoggerContext, @@ -70,7 +70,7 @@ object LogbackConfiguration extends LazyLogging { layoutEncoder.start() val fileAppender = new FileAppender[ILoggingEvent] - fileAppender.setFile(logPath) + fileAppender.setFile(logPath.toString) fileAppender.setEncoder(layoutEncoder) fileAppender.setContext(loggerContext) fileAppender.setName(appenderName) diff --git a/src/main/scala/edu/ie3/simona/main/RunSimona.scala b/src/main/scala/edu/ie3/simona/main/RunSimona.scala index b08db66361..3ba0c8f9aa 100644 --- a/src/main/scala/edu/ie3/simona/main/RunSimona.scala +++ b/src/main/scala/edu/ie3/simona/main/RunSimona.scala @@ -57,7 +57,7 @@ trait RunSimona[T <: SimonaSetup] extends LazyLogging { private def printGoodbye( successful: Boolean, - outputPath: String = "", + logOutputDir: Path, ): Unit = { val myWords = Array( "\"Vielleicht ist heute ein besonders guter Tag zum Sterben.\" - Worf (in Star Trek: Der erste Kontakt)", @@ -78,7 +78,7 @@ trait RunSimona[T <: SimonaSetup] extends LazyLogging { // to ensure that the link to the log is printed last Thread.sleep(1000) - val path = Path.of(outputPath).resolve("simona.log").toUri + val path = logOutputDir.resolve("simona.log").toUri logger.error( s"Simulation stopped due to the occurrence of an error! The full log can be found here: $path" diff --git a/src/main/scala/edu/ie3/simona/sim/setup/SetupHelper.scala b/src/main/scala/edu/ie3/simona/sim/setup/SetupHelper.scala index d5b21a99a7..23fe9e1997 100644 --- a/src/main/scala/edu/ie3/simona/sim/setup/SetupHelper.scala +++ b/src/main/scala/edu/ie3/simona/sim/setup/SetupHelper.scala @@ -220,7 +220,7 @@ trait SetupHelper extends LazyLogging { val modelsToWrite = SetupHelper.allResultEntitiesToWrite(simonaConfig.simona.output) - val resultFileHierarchy = ResultFileHierarchy( + ResultFileHierarchy( simonaConfig.simona.output.base.dir, simonaConfig.simona.simulationName, ResultEntityPathConfig( @@ -230,15 +230,10 @@ trait SetupHelper extends LazyLogging { simonaConfig.simona.simulationName, ), ), + config = Some(config), addTimeStampToOutputDir = simonaConfig.simona.output.base.addTimestampToOutputDir, - createDirs = createDirs, ) - - // copy config data to output directory - ResultFileHierarchy.prepareDirectories(config, resultFileHierarchy) - - resultFileHierarchy } } diff --git a/src/main/scala/edu/ie3/simona/sim/setup/SimonaSetup.scala b/src/main/scala/edu/ie3/simona/sim/setup/SimonaSetup.scala index 4a2d8abd42..4c452a4c3c 100644 --- a/src/main/scala/edu/ie3/simona/sim/setup/SimonaSetup.scala +++ b/src/main/scala/edu/ie3/simona/sim/setup/SimonaSetup.scala @@ -21,6 +21,8 @@ import org.apache.pekko.actor.typed.ActorRef import org.apache.pekko.actor.typed.scaladsl.ActorContext import org.apache.pekko.actor.{ActorRef => ClassicRef} +import java.nio.file.Path + /** Trait that can be used to set up a customized simona simulation by providing * implementations for all setup information required by a * [[edu.ie3.simona.sim.SimonaSim]]. Most of the time, using or extending @@ -40,7 +42,7 @@ trait SimonaSetup { /** Directory of the log output. */ - def logOutputDir: String + def logOutputDir: Path /** Creates the runtime event listener * diff --git a/src/main/scala/edu/ie3/simona/sim/setup/SimonaStandaloneSetup.scala b/src/main/scala/edu/ie3/simona/sim/setup/SimonaStandaloneSetup.scala index 65d47863cf..55848d2ced 100644 --- a/src/main/scala/edu/ie3/simona/sim/setup/SimonaStandaloneSetup.scala +++ b/src/main/scala/edu/ie3/simona/sim/setup/SimonaStandaloneSetup.scala @@ -50,6 +50,7 @@ import org.apache.pekko.actor.typed.scaladsl.adapter.{ } import org.apache.pekko.actor.{ActorRef => ClassicRef} +import java.nio.file.Path import java.util.UUID import java.util.concurrent.LinkedBlockingQueue import scala.jdk.CollectionConverters._ @@ -68,7 +69,7 @@ class SimonaStandaloneSetup( override val args: Array[String], ) extends SimonaSetup { - override def logOutputDir: String = resultFileHierarchy.logOutputDir + override def logOutputDir: Path = resultFileHierarchy.logOutputDir override def gridAgents( context: ActorContext[_], diff --git a/src/main/scala/edu/ie3/simona/util/ResultFileHierarchy.scala b/src/main/scala/edu/ie3/simona/util/ResultFileHierarchy.scala index 3c1028be2b..3dc207d350 100644 --- a/src/main/scala/edu/ie3/simona/util/ResultFileHierarchy.scala +++ b/src/main/scala/edu/ie3/simona/util/ResultFileHierarchy.scala @@ -7,7 +7,7 @@ package edu.ie3.simona.util import java.io.{BufferedWriter, File, FileWriter} -import java.nio.file.{Files, Paths} +import java.nio.file.{Files, Path, Paths} import java.text.SimpleDateFormat import com.typesafe.config.{ConfigRenderOptions, Config => TypesafeConfig} import com.typesafe.scalalogging.LazyLogging @@ -20,130 +20,135 @@ import edu.ie3.simona.exceptions.FileHierarchyException import edu.ie3.simona.io.result.ResultSinkType import edu.ie3.simona.io.result.ResultSinkType.Csv import edu.ie3.simona.logging.logback.LogbackConfiguration -import edu.ie3.simona.util.ResultFileHierarchy.ResultEntityPathConfig import edu.ie3.util.io.FileIOUtils import org.apache.commons.io.FilenameUtils._ import scala.jdk.OptionConverters.RichOptional -/** Represents the output directory where the results will be materialized. If - * new directories are added please remember to add them to the dirsToBeCreated - * Vector if they should be created. Otherwise, they will not be created! - * - * @version 0.1 - * @since 12.01.20 +/** Represents the output directory where the results will be materialized. */ -final case class ResultFileHierarchy( - private val outputDir: String, - private val simulationName: String, - private val resultEntityPathConfig: ResultEntityPathConfig, - private val configureLogger: String => Unit = LogbackConfiguration.default, - private val addTimeStampToOutputDir: Boolean = true, - private val createDirs: Boolean = false, -) extends LazyLogging { +final case class ResultFileHierarchy private ( + runOutputDir: Path, + rawOutputDataFilePaths: Map[Class[_ <: ResultEntity], Path], + configOutputDir: Path, + logOutputDir: Path, + tmpDir: Path, + resultSinkType: ResultSinkType, + resultEntitiesToConsider: Set[Class[_ <: ResultEntity]], +) - private val fileSeparator: String = File.separator - - val runStartTimeUTC: String = - new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss").format(new java.util.Date()) - - val baseOutputDir: String = buildBaseOutputDir - - val runOutputDir: String = buildRunOutputDir +object ResultFileHierarchy extends LazyLogging { - val configOutputDir: String = - runOutputDir.concat(fileSeparator).concat("configs") + /** Creates the [[ResultFileHierarchy]] and relevant directories + */ + def apply( + outputDir: String, + simulationName: String, + resultEntityPathConfig: ResultEntityPathConfig, + configureLogger: Path => Unit = LogbackConfiguration.default, + config: Option[TypesafeConfig] = None, + addTimeStampToOutputDir: Boolean = true, + ): ResultFileHierarchy = { + + val runStartTimeUTC = Option.when(addTimeStampToOutputDir)( + new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss").format(new java.util.Date()) + ) - val rawOutputDataDir: String = - runOutputDir.concat(fileSeparator).concat("rawOutputData") + val baseOutputDir = buildBaseOutputDir(outputDir) - val logOutputDir: String = - runOutputDir.concat(fileSeparator).concat("log") + val runOutputDir = buildRunOutputDir( + baseOutputDir, + simulationName, + runStartTimeUTC, + ) - val resultSinkType: ResultSinkType = resultEntityPathConfig.resultSinkType + val configOutputDir = runOutputDir.resolve("configs") + val rawOutputDataDir = runOutputDir.resolve("rawOutputData") + val logOutputDir = runOutputDir.resolve("log") + val tmpDir = runOutputDir.resolve("tmp") - val resultEntitiesToConsider: Set[Class[_ <: ResultEntity]] = - resultEntityPathConfig.resultEntitiesToConsider + val resultSinkType: ResultSinkType = resultEntityPathConfig.resultSinkType - val rawOutputDataFilePaths: Map[Class[_ <: ResultEntity], String] = { - resultSinkType match { - case csv: Csv => - resultEntityPathConfig.resultEntitiesToConsider - .map(resultEntityClass => - ( - resultEntityClass, - ResultFileHierarchy.buildRawOutputFilePath( + val rawOutputDataFilePaths: Map[Class[_ <: ResultEntity], Path] = { + resultSinkType match { + case csv: Csv => + resultEntityPathConfig.resultEntitiesToConsider + .map(resultEntityClass => + ( resultEntityClass, - csv, - rawOutputDataDir, - fileSeparator, - ), + ResultFileHierarchy.buildRawOutputFilePath( + resultEntityClass, + csv, + rawOutputDataDir, + ), + ) ) - ) - .toMap - case _ => - Map.empty + .toMap + case _ => + Map.empty + } } - } - - val graphOutputDir: String = - runOutputDir.concat(fileSeparator).concat("graphs") - val kpiOutputDir: String = runOutputDir.concat(fileSeparator).concat("kpi") - - val tmpDir: String = runOutputDir.concat(fileSeparator).concat("tmp") + val dirsToBeCreated = Seq( + baseOutputDir, + runOutputDir, + configOutputDir, + rawOutputDataDir, + logOutputDir, + tmpDir, + ) - private val dirsToBeCreated: Vector[String] = Vector( - baseOutputDir, - runOutputDir, - configOutputDir, - rawOutputDataDir, - graphOutputDir, - kpiOutputDir, - tmpDir, - logOutputDir, - ) + val resultFileHierarchy = ResultFileHierarchy( + runOutputDir, + rawOutputDataFilePaths, + configOutputDir, + logOutputDir, + tmpDir, + resultSinkType, + resultEntityPathConfig.resultEntitiesToConsider, + ) + prepareDirectories( + baseOutputDir, + dirsToBeCreated, + resultFileHierarchy, + config, + ) - // needs to be the latest call because otherwise the values are null as they are not initialized yet - if (createDirs) - ResultFileHierarchy.createOutputDirectories(this) + // needs to be done after dir creation + configureLogger(logOutputDir) - // needs to be done after dir creation - configureLogger(logOutputDir) + resultFileHierarchy + } - /** Builds the base output directory string + /** Builds the base output directory * * @return - * the filepath string to the directory + * the filepath of the directory */ - private def buildBaseOutputDir: String = { - + private def buildBaseOutputDir( + outputDir: String + ): Path = { // clean file string if necessary val cleanedBaseOutputDir = { val normalizedOutputDir = normalize(outputDir) - (fileSeparator + "$").r.replaceAllIn(normalizedOutputDir, "") + (File.separator + "$").r.replaceAllIn(normalizedOutputDir, "") } - // create base output dir if non-existent - Paths.get(cleanedBaseOutputDir).toFile.getAbsolutePath + Paths.get(cleanedBaseOutputDir) } - /** Builds the output directory string for this specific run - * - * @return + /** Builds the output directory for this specific run */ - private def buildRunOutputDir: String = { + private def buildRunOutputDir( + baseOutputDir: Path, + simulationName: String, + runStartTimeUTC: Option[String], + ): Path = { val optionalSuffix = - if (addTimeStampToOutputDir) s"_$runStartTimeUTC" else "" - baseOutputDir - .concat(fileSeparator) - .concat(simulationName) - .concat(optionalSuffix) - } + runStartTimeUTC.map(pattern => s"_$pattern").getOrElse("") -} - -object ResultFileHierarchy extends LazyLogging { + baseOutputDir.resolve(s"$simulationName$optionalSuffix") + } /** @param resultEntitiesToConsider * [[ResultEntity]] s to consider to be written out @@ -161,8 +166,6 @@ object ResultFileHierarchy extends LazyLogging { * the csv sink type parameters * @param rawOutputDataDir * the directory of the raw output data - * @param fileSeparator - * the file separator to be used * @return * an absolute file path as string for the provided model class incl. file * name + extension @@ -170,9 +173,8 @@ object ResultFileHierarchy extends LazyLogging { private def buildRawOutputFilePath( modelClass: Class[_ <: ResultEntity], csvSink: Csv, - rawOutputDataDir: String, - fileSeparator: String, - ): String = { + rawOutputDataDir: Path, + ): Path = { val fileEnding = if (csvSink.fileFormat.startsWith(".")) csvSink.fileFormat @@ -192,64 +194,72 @@ object ResultFileHierarchy extends LazyLogging { ) } - rawOutputDataDir - .concat(fileSeparator) - .concat(filename.toString) - .concat(fileEnding) + rawOutputDataDir.resolve(s"${filename.toString}$fileEnding") } /** Prepares the output directories to be ready to hold the output data. This * includes creating the run directory with all subsequent directories as * well as copying the simulation configuration to the output dir * - * @param config + * @param baseOutputDir + * The base output directory + * @param dirsToBeCreated + * The directories that need to be created + * @param maybeConfig * the config of the current simulation * @param resultFileHierarchy * the output file hierarchy of the current simulation */ - def prepareDirectories( - config: TypesafeConfig, + private def prepareDirectories( + baseOutputDir: Path, + dirsToBeCreated: Seq[Path], resultFileHierarchy: ResultFileHierarchy, + maybeConfig: Option[TypesafeConfig], ): Unit = { // create output directories if they are not present yet if (!runOutputDirExists(resultFileHierarchy)) - ResultFileHierarchy.createOutputDirectories(resultFileHierarchy) + createOutputDirectories( + baseOutputDir, + dirsToBeCreated, + resultFileHierarchy, + ) - logger.info( - "Processing configs for simulation: {}.", - config.getString("simona.simulationName"), - ) + maybeConfig.foreach { config => + logger.info( + "Processing configs for simulation: {}.", + config.getString("simona.simulationName"), + ) - val outFile = - Paths.get(resultFileHierarchy.configOutputDir, "vn_simona.conf").toFile - val bw = new BufferedWriter(new FileWriter(outFile)) - bw.write( - config - .root() - .render( - ConfigRenderOptions - .defaults() - .setOriginComments(false) - .setComments(false) - ) - ) - bw.close() - logger.info("Config '{}' written to '{}'.", outFile.getPath, outFile) + val outFile = + resultFileHierarchy.configOutputDir.resolve("vn_simona.conf").toFile + val bw = new BufferedWriter(new FileWriter(outFile)) + bw.write( + config + .root() + .render( + ConfigRenderOptions + .defaults() + .setOriginComments(false) + .setComments(false) + ) + ) + bw.close() + logger.info("Config '{}' written to '{}'.", outFile.getPath, outFile) + } } /** Checks if the directory of the current run already exists * - * @param outputFileHierarchy + * @param fileHierarchy * the [[ResultFileHierarchy]] that holds information on the run directory * path * @return * true if it exists, false if not */ - def runOutputDirExists(outputFileHierarchy: ResultFileHierarchy): Boolean = { - new File(outputFileHierarchy.runOutputDir).exists() && new File( - outputFileHierarchy.runOutputDir - ).listFiles().length > 0 + def runOutputDirExists(fileHierarchy: ResultFileHierarchy): Boolean = { + val outputDir = fileHierarchy.runOutputDir.toFile + outputDir.exists() && outputDir.listFiles().length > 0 } /** Creates all output directories of the provided [[ResultFileHierarchy]] @@ -257,11 +267,12 @@ object ResultFileHierarchy extends LazyLogging { * @param outputFileHierarchy * the [[ResultFileHierarchy]] the directories should be created for */ - def createOutputDirectories( - outputFileHierarchy: ResultFileHierarchy + private def createOutputDirectories( + baseOutputDir: Path, + dirsToBeCreated: Seq[Path], + outputFileHierarchy: ResultFileHierarchy, ): Unit = { // try to create base output dir - val baseOutputDir = Paths.get(outputFileHierarchy.baseOutputDir) // / check for existence of the provided baseOutputDir, if not create it if (Files.exists(baseOutputDir) && baseOutputDir.toFile.isFile) { throw new FileHierarchyException( @@ -270,17 +281,17 @@ object ResultFileHierarchy extends LazyLogging { } // check if there is data inside the runOutputDir taking into account the provided FileHandling - val runOutputDir = new File(outputFileHierarchy.runOutputDir) + val runOutputDir = outputFileHierarchy.runOutputDir.toFile if (runOutputDir.exists() && runOutputDir.listFiles().length > 0) { // files inside the runOutputDir -> fail throw new FileHierarchyException( - s"The runOutputDir ${outputFileHierarchy.runOutputDir} already exists and is NOT empty! " + + s"The runOutputDir ${outputFileHierarchy.runOutputDir.toString} already exists and is NOT empty! " + s"Please either delete or empty the directory." ) } // create the output directories for the specific run - outputFileHierarchy.dirsToBeCreated.foreach(createDir) + dirsToBeCreated.foreach(createDir) } @@ -289,8 +300,8 @@ object ResultFileHierarchy extends LazyLogging { * @param dir * the full path where the directory should be created (incl. it's name) */ - private def createDir(dir: String): Unit = { - val dirFile = new File(dir) + private def createDir(dir: Path): Unit = { + val dirFile = dir.toFile if (!dirFile.mkdirs() && !dirFile.exists()) throw new FileHierarchyException( "The output directory path " + dir diff --git a/src/test/scala/edu/ie3/simona/event/listener/ResultEventListenerSpec.scala b/src/test/scala/edu/ie3/simona/event/listener/ResultEventListenerSpec.scala index 26c3145945..68caa1706b 100644 --- a/src/test/scala/edu/ie3/simona/event/listener/ResultEventListenerSpec.scala +++ b/src/test/scala/edu/ie3/simona/event/listener/ResultEventListenerSpec.scala @@ -84,7 +84,6 @@ class ResultEventListenerSpec classes, resultSinkType, ), - createDirs = true, ) } @@ -122,14 +121,14 @@ class ResultEventListenerSpec ) // after the creation of the listener, it is expected that a corresponding raw result data file is present - val outputFile = new File( - fileHierarchy.rawOutputDataFilePaths.getOrElse( + val outputFile = fileHierarchy.rawOutputDataFilePaths + .getOrElse( classOf[PvResult], fail( s"Cannot get filepath for raw result file of class '${classOf[PvResult].getSimpleName}' from outputFileHierarchy!'" ), ) - ) + .toFile assert(outputFile.exists) assert(outputFile.isFile) @@ -162,14 +161,14 @@ class ResultEventListenerSpec listenerRef ! ParticipantResultEvent(dummyPvResult) - val outputFile = new File( - specificOutputFileHierarchy.rawOutputDataFilePaths.getOrElse( + val outputFile = specificOutputFileHierarchy.rawOutputDataFilePaths + .getOrElse( classOf[PvResult], fail( s"Cannot get filepath for raw result file of class '${classOf[PvResult].getSimpleName}' from outputFileHierarchy!'" ), ) - ) + .toFile // wait until output file exists (headers are flushed out immediately): awaitCond( @@ -219,39 +218,37 @@ class ResultEventListenerSpec ) val outputFiles = Map( - dummyNodeResultString -> new File( - specificOutputFileHierarchy.rawOutputDataFilePaths.getOrElse( + dummyNodeResultString -> specificOutputFileHierarchy.rawOutputDataFilePaths + .getOrElse( classOf[NodeResult], fail( s"Cannot get filepath for raw result file of class '${classOf[NodeResult].getSimpleName}' from outputFileHierarchy!'" ), - ) - ), - dummySwitchResultString -> new File( + ), + dummySwitchResultString -> specificOutputFileHierarchy.rawOutputDataFilePaths.getOrElse( classOf[SwitchResult], fail( s"Cannot get filepath for raw result file of class '${classOf[SwitchResult].getSimpleName}' from outputFileHierarchy!'" ), - ) - ), - dummyLineResultDataString -> new File( - specificOutputFileHierarchy.rawOutputDataFilePaths.getOrElse( + ), + dummyLineResultDataString -> specificOutputFileHierarchy.rawOutputDataFilePaths + .getOrElse( classOf[LineResult], fail( s"Cannot get filepath for raw result file of class '${classOf[LineResult].getSimpleName}' from outputFileHierarchy!'" ), - ) - ), - dummyTrafo2wResultDataString -> new File( - specificOutputFileHierarchy.rawOutputDataFilePaths.getOrElse( + ), + dummyTrafo2wResultDataString -> specificOutputFileHierarchy.rawOutputDataFilePaths + .getOrElse( classOf[Transformer2WResult], fail( s"Cannot get filepath for raw result file of class '${classOf[Transformer2WResult].getSimpleName}' from outputFileHierarchy!'" ), - ) - ), - ) + ), + ).map { case (dummyString, path) => + (dummyString, path.toFile) + } // wait until all output files exist (headers are flushed out immediately): awaitCond( @@ -308,14 +305,15 @@ class ResultEventListenerSpec ) ) - val outputFile = new File( - fileHierarchy.rawOutputDataFilePaths.getOrElse( + val outputFile = fileHierarchy.rawOutputDataFilePaths + .getOrElse( classOf[Transformer3WResult], fail( s"Cannot get filepath for raw result file of class '${classOf[Transformer3WResult].getSimpleName}' from outputFileHierarchy!'" ), ) - ) + .toFile + /* The result file is created at start up and only contains a headline. */ awaitCond( outputFile.exists(), @@ -386,12 +384,14 @@ class ResultEventListenerSpec val outputFile = new File( ".gz$".r.replaceAllIn( - specificOutputFileHierarchy.rawOutputDataFilePaths.getOrElse( - classOf[PvResult], - fail( - s"Cannot get filepath for raw result file of class '${classOf[PvResult].getSimpleName}' from outputFileHierarchy!'" - ), - ), + specificOutputFileHierarchy.rawOutputDataFilePaths + .getOrElse( + classOf[PvResult], + fail( + s"Cannot get filepath for raw result file of class '${classOf[PvResult].getSimpleName}' from outputFileHierarchy!'" + ), + ) + .toString, "", ) ) @@ -413,26 +413,29 @@ class ResultEventListenerSpec // wait until file exists awaitCond( - new File( - specificOutputFileHierarchy.rawOutputDataFilePaths.getOrElse( + specificOutputFileHierarchy.rawOutputDataFilePaths + .getOrElse( classOf[PvResult], fail( s"Cannot get filepath for raw result file of class '${classOf[PvResult].getSimpleName}' from outputFileHierarchy!'" ), ) - ).exists, + .toFile + .exists, timeoutDuration, ) val resultFileSource = Source.fromInputStream( new GZIPInputStream( new FileInputStream( - specificOutputFileHierarchy.rawOutputDataFilePaths.getOrElse( - classOf[PvResult], - fail( - s"Cannot get filepath for raw result file of class '${classOf[PvResult].getSimpleName}' from outputFileHierarchy!'" - ), - ) + specificOutputFileHierarchy.rawOutputDataFilePaths + .getOrElse( + classOf[PvResult], + fail( + s"Cannot get filepath for raw result file of class '${classOf[PvResult].getSimpleName}' from outputFileHierarchy!'" + ), + ) + .toFile ) ) ) diff --git a/src/test/scala/edu/ie3/simona/integration/RunSimonaStandaloneIT.scala b/src/test/scala/edu/ie3/simona/integration/RunSimonaStandaloneIT.scala index 4e53f0e078..083b9681e3 100644 --- a/src/test/scala/edu/ie3/simona/integration/RunSimonaStandaloneIT.scala +++ b/src/test/scala/edu/ie3/simona/integration/RunSimonaStandaloneIT.scala @@ -90,7 +90,7 @@ class RunSimonaStandaloneIT /* check the results */ // check configs - val configOutputDir = new File(resultFileHierarchy.configOutputDir) + val configOutputDir = resultFileHierarchy.configOutputDir.toFile configOutputDir.isDirectory shouldBe true configOutputDir.listFiles.toVector.size shouldBe 1 @@ -118,10 +118,12 @@ class RunSimonaStandaloneIT entityClass: Class[_ <: ResultEntity], ): BufferedSource = { Source.fromFile( - resultFileHierarchy.rawOutputDataFilePaths.getOrElse( - entityClass, - fail(s"Unable to get output path for result entity: $entityClass"), - ) + resultFileHierarchy.rawOutputDataFilePaths + .getOrElse( + entityClass, + fail(s"Unable to get output path for result entity: $entityClass"), + ) + .toFile ) } diff --git a/src/test/scala/edu/ie3/simona/io/file/ResultFileHierarchySpec.scala b/src/test/scala/edu/ie3/simona/io/file/ResultFileHierarchySpec.scala index 3b89d418f2..c273634fbe 100644 --- a/src/test/scala/edu/ie3/simona/io/file/ResultFileHierarchySpec.scala +++ b/src/test/scala/edu/ie3/simona/io/file/ResultFileHierarchySpec.scala @@ -51,37 +51,15 @@ class ResultFileHierarchySpec ), ) - val runOutputDirWithDate = - "vn_simona_".concat(validOutputFileHierarchy.runStartTimeUTC) - - relativizePath( - validOutputFileHierarchy.baseOutputDir - ).toString shouldBe baseOutputDir - relativizePath( - validOutputFileHierarchy.runOutputDir - ).toString shouldBe baseOutputDir + fileSeparator + runOutputDirWithDate - relativizePath( - validOutputFileHierarchy.tmpDir - ).toString shouldBe baseOutputDir + fileSeparator + runOutputDirWithDate + fileSeparator + "tmp" - relativizePath( - validOutputFileHierarchy.configOutputDir - ).toString shouldBe baseOutputDir + fileSeparator + runOutputDirWithDate + fileSeparator + "configs" - relativizePath( - validOutputFileHierarchy.rawOutputDataDir - ).toString shouldBe baseOutputDir + fileSeparator + runOutputDirWithDate + fileSeparator + "rawOutputData" - relativizePath( - validOutputFileHierarchy.graphOutputDir - ).toString shouldBe baseOutputDir + fileSeparator + runOutputDirWithDate + fileSeparator + "graphs" - relativizePath( - validOutputFileHierarchy.kpiOutputDir - ).toString shouldBe baseOutputDir + fileSeparator + runOutputDirWithDate + fileSeparator + "kpi" - - relativizePath( - validOutputFileHierarchy.rawOutputDataFilePaths(classOf[PvResult]) - ).toString shouldBe baseOutputDir + fileSeparator + runOutputDirWithDate + fileSeparator + "rawOutputData" + fileSeparator + "pref_pv_res_suff.csv" + validOutputFileHierarchy.tmpDir.toString shouldBe validOutputFileHierarchy.runOutputDir.toString + fileSeparator + "tmp" + validOutputFileHierarchy.configOutputDir.toString shouldBe validOutputFileHierarchy.runOutputDir.toString + fileSeparator + "configs" + validOutputFileHierarchy.logOutputDir.toString shouldBe validOutputFileHierarchy.runOutputDir.toString + fileSeparator + "log" + + validOutputFileHierarchy + .rawOutputDataFilePaths(classOf[PvResult]) + .toString shouldBe validOutputFileHierarchy.runOutputDir.toString + fileSeparator + "rawOutputData" + fileSeparator + "pref_pv_res_suff.csv" } - "not write directories automatically on instantiation" in {} // todo "write directories automatically on instantiation when requested so" in { // delete file if they exist @@ -97,7 +75,6 @@ class ResultFileHierarchySpec Set(classOf[PvResult]), ResultSinkType.Csv("csv", "pref", "suff"), ), - createDirs = true, ) // check for existence of run output dir @@ -106,26 +83,13 @@ class ResultFileHierarchySpec ) shouldBe true // check for existence of other folders - assert( - Files.exists(new File(validOutputFileHierarchy.baseOutputDir).toPath) - ) - assert(Files.exists(new File(validOutputFileHierarchy.tmpDir).toPath)) - assert( - Files.exists(new File(validOutputFileHierarchy.configOutputDir).toPath) - ) - assert( - Files.exists(new File(validOutputFileHierarchy.rawOutputDataDir).toPath) - ) - assert( - Files.exists(new File(validOutputFileHierarchy.graphOutputDir).toPath) - ) - assert( - Files.exists(new File(validOutputFileHierarchy.kpiOutputDir).toPath) - ) + assert(Files.exists(validOutputFileHierarchy.configOutputDir)) + assert(Files.exists(validOutputFileHierarchy.logOutputDir)) + assert(Files.exists(validOutputFileHierarchy.tmpDir)) // check if tmp directory can be deleted by output file hierarchy ResultFileHierarchy.deleteTmpDir(validOutputFileHierarchy) - assert(!Files.exists(new File(validOutputFileHierarchy.tmpDir).toPath)) + assert(!Files.exists(validOutputFileHierarchy.tmpDir)) } @@ -133,10 +97,8 @@ class ResultFileHierarchySpec } - private def relativizePath(fullPath: String): Path = { - new File(new File("").getAbsolutePath).toPath - .relativize(new File(fullPath).toPath) - } + private def relativizePath(fullPath: Path): Path = + new File(new File("").getAbsolutePath).toPath.relativize(fullPath) // todo output model path config compression should always be disabled -> test for this diff --git a/src/test/scala/edu/ie3/simona/sim/SimonaSimSpec.scala b/src/test/scala/edu/ie3/simona/sim/SimonaSimSpec.scala index 63be126c0f..9d1140be17 100644 --- a/src/test/scala/edu/ie3/simona/sim/SimonaSimSpec.scala +++ b/src/test/scala/edu/ie3/simona/sim/SimonaSimSpec.scala @@ -34,6 +34,7 @@ import org.apache.pekko.actor.typed.scaladsl.{ActorContext, Behaviors} import org.apache.pekko.actor.typed.{ActorRef, Behavior} import org.apache.pekko.actor.{ActorRef => ClassicRef} +import java.nio.file.Path import java.util.UUID class SimonaSimSpec extends ScalaTestWithActorTestKit with UnitSpec { @@ -398,7 +399,7 @@ object SimonaSimSpec { override val args: Array[String] = Array.empty[String] - override def logOutputDir: String = throw new NotImplementedError() + override def logOutputDir: Path = throw new NotImplementedError() override def runtimeEventListener( context: ActorContext[_] diff --git a/src/test/scala/edu/ie3/simona/sim/setup/SimonaSetupSpec.scala b/src/test/scala/edu/ie3/simona/sim/setup/SimonaSetupSpec.scala index 7fa3ab3469..0428ed765b 100644 --- a/src/test/scala/edu/ie3/simona/sim/setup/SimonaSetupSpec.scala +++ b/src/test/scala/edu/ie3/simona/sim/setup/SimonaSetupSpec.scala @@ -26,13 +26,14 @@ import org.apache.pekko.actor.typed.ActorRef import org.apache.pekko.actor.typed.scaladsl.ActorContext import org.apache.pekko.actor.{ActorRef => ClassicRef} +import java.nio.file.Path import java.util.UUID class SimonaSetupSpec extends UnitSpec with SimonaSetup with SubGridGateMokka { override val args: Array[String] = Array.empty[String] - override def logOutputDir: String = throw new NotImplementedError() + override def logOutputDir: Path = throw new NotImplementedError() override def runtimeEventListener( context: ActorContext[_] From 6e821bf84454aafc298b7059a7e15254f852892e Mon Sep 17 00:00:00 2001 From: Sebastian Peter Date: Mon, 18 Nov 2024 21:15:44 +0100 Subject: [PATCH 2/3] Adding to changelog Signed-off-by: Sebastian Peter --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4cef76459a..4ea36f463e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -99,6 +99,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Updated `ExtEvSimulationClasses` [#898](https://github.com/ie3-institute/simona/issues/898) - Refactoring of `ThermalGrid.energyGrid` to distinguish between demand of house and storage [#928](https://github.com/ie3-institute/simona/issues/928) - Refactoring to use zeroKW and zeroKWH in thermal grid unit tests [#1023](https://github.com/ie3-institute/simona/issues/1023) +- Refactor `ResultFileHierarchy` [#1031](https://github.com/ie3-institute/simona/issues/1031) ### Fixed - Fix rendering of references in documentation [#505](https://github.com/ie3-institute/simona/issues/505) From c0d45979ffb8b7ad2e4f8adad516435e63adefb1 Mon Sep 17 00:00:00 2001 From: Sebastian Peter Date: Tue, 19 Nov 2024 10:52:46 +0100 Subject: [PATCH 3/3] Addressing reviewer's comments Signed-off-by: Sebastian Peter --- src/main/scala/edu/ie3/simona/sim/setup/SetupHelper.scala | 8 ++------ .../edu/ie3/simona/io/file/ResultFileHierarchySpec.scala | 3 --- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/src/main/scala/edu/ie3/simona/sim/setup/SetupHelper.scala b/src/main/scala/edu/ie3/simona/sim/setup/SetupHelper.scala index 05a6f0ffd5..1b2d2df7e7 100644 --- a/src/main/scala/edu/ie3/simona/sim/setup/SetupHelper.scala +++ b/src/main/scala/edu/ie3/simona/sim/setup/SetupHelper.scala @@ -202,17 +202,13 @@ trait SetupHelper extends LazyLogging { /** Build the result file hierarchy based on the provided configuration file. * The provided type safe config must be able to be parsed as * [[SimonaConfig]], otherwise an exception is thrown + * * @param config * the configuration file - * @param createDirs - * if directories of the result file hierarchy should be created or not * @return * the resulting result file hierarchy */ - def buildResultFileHierarchy( - config: TypesafeConfig, - createDirs: Boolean = true, - ): ResultFileHierarchy = { + def buildResultFileHierarchy(config: TypesafeConfig): ResultFileHierarchy = { val simonaConfig = SimonaConfig(config) diff --git a/src/test/scala/edu/ie3/simona/io/file/ResultFileHierarchySpec.scala b/src/test/scala/edu/ie3/simona/io/file/ResultFileHierarchySpec.scala index c273634fbe..6ddc02a059 100644 --- a/src/test/scala/edu/ie3/simona/io/file/ResultFileHierarchySpec.scala +++ b/src/test/scala/edu/ie3/simona/io/file/ResultFileHierarchySpec.scala @@ -97,9 +97,6 @@ class ResultFileHierarchySpec } - private def relativizePath(fullPath: Path): Path = - new File(new File("").getAbsolutePath).toPath.relativize(fullPath) - // todo output model path config compression should always be disabled -> test for this }