diff --git a/spark-jobs/src/main/scala/za/co/absa/enceladus/common/CommonJobExecution.scala b/spark-jobs/src/main/scala/za/co/absa/enceladus/common/CommonJobExecution.scala index 4150665fa..5ba0b901d 100644 --- a/spark-jobs/src/main/scala/za/co/absa/enceladus/common/CommonJobExecution.scala +++ b/spark-jobs/src/main/scala/za/co/absa/enceladus/common/CommonJobExecution.scala @@ -275,8 +275,7 @@ trait CommonJobExecution extends ProjectMetadata { case Some(version) => version case None => - // publishFs for this specific feature (needed for missing reportVersion until reusable - // common "PathConfig" with FS objects is established) + // Since `pathConfig.publish.fileSystem` is not available at this point yet, a temporary publish-FS is create & used here instead val tempPublishFs: FileSystem = FileSystemUtils.getFileSystemFromPath(dataset.hdfsPublishPath) val fsUtils = HadoopFsUtils.getOrCreate(tempPublishFs) val newVersion = fsUtils.getLatestVersion(dataset.hdfsPublishPath, jobConfig.reportDate) + 1 diff --git a/utils/src/main/scala/za/co/absa/enceladus/utils/fs/HadoopFsUtils.scala b/utils/src/main/scala/za/co/absa/enceladus/utils/fs/HadoopFsUtils.scala index 4dd7de05a..534982c77 100644 --- a/utils/src/main/scala/za/co/absa/enceladus/utils/fs/HadoopFsUtils.scala +++ b/utils/src/main/scala/za/co/absa/enceladus/utils/fs/HadoopFsUtils.scala @@ -182,9 +182,7 @@ class HadoopFsUtils private()(implicit fs: FileSystem) extends DistributedFsUtil tmpFile.deleteOnExit() FileUtils.writeByteArrayToFile(tmpFile, content) tmpFile.getAbsolutePath - - // why not use - // fs.copyToLocalFile(false, new Path(hdfsPath), new Path("someLocalName"), true) + //TODO #1645 Possible simplification of this code } override def read(distPath: String): String = {