diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/SimpleFileCopyStrategy.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/SimpleFileCopyStrategy.groovy index 427f6a84d3..e5d7262c97 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/SimpleFileCopyStrategy.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/SimpleFileCopyStrategy.groovy @@ -168,14 +168,17 @@ class SimpleFileCopyStrategy implements ScriptFileCopyStrategy { final escape = new ArrayList(outputFiles.size()) for( String it : patterns ) - escape.add( Escape.path(it) ) + escape.add( Escape.path( it, true ) ) final mode = stageoutMode ?: ( workDir==targetDir ? 'copy' : 'move' ) return """\ IFS=\$'\\n' - for name in \$(eval "ls -1d ${escape.join(' ')}" | sort | uniq); do + pathes=`ls -1d ${escape.join(' ')} | sort | uniq` + set -f + for name in \$pathes; do ${stageOutCommand('$name', targetDir, mode)} || true done + set +f unset IFS""".stripIndent(true) } diff --git a/modules/nextflow/src/test/groovy/nextflow/executor/BashWrapperBuilderTest.groovy b/modules/nextflow/src/test/groovy/nextflow/executor/BashWrapperBuilderTest.groovy index d2dbadbc0f..42dbee9a21 100644 --- a/modules/nextflow/src/test/groovy/nextflow/executor/BashWrapperBuilderTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/executor/BashWrapperBuilderTest.groovy @@ -408,9 +408,12 @@ class BashWrapperBuilderTest extends Specification { then: binding.unstage_outputs == '''\ IFS=$'\\n' - for name in $(eval "ls -1d test.bam test.bai" | sort | uniq); do + pathes=`ls -1d test.bam test.bai | sort | uniq` + set -f + for name in $pathes; do nxf_fs_copy "$name" /work/dir || true done + set +f unset IFS '''.stripIndent().rightTrim() @@ -425,9 +428,12 @@ class BashWrapperBuilderTest extends Specification { then: binding.unstage_outputs == '''\ IFS=$'\\n' - for name in $(eval "ls -1d test.bam test.bai" | sort | uniq); do + pathes=`ls -1d test.bam test.bai | sort | uniq` + set -f + for name in $pathes; do nxf_fs_move "$name" /another/dir || true done + set +f unset IFS '''.stripIndent().rightTrim() } diff --git a/modules/nextflow/src/test/groovy/nextflow/executor/SimpleFileCopyStrategyTest.groovy b/modules/nextflow/src/test/groovy/nextflow/executor/SimpleFileCopyStrategyTest.groovy index f35f2e63a5..24372c9070 100644 --- a/modules/nextflow/src/test/groovy/nextflow/executor/SimpleFileCopyStrategyTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/executor/SimpleFileCopyStrategyTest.groovy @@ -16,10 +16,10 @@ */ package nextflow.executor - - +import java.nio.file.Files +import java.nio.file.Path import java.nio.file.Paths - +import java.util.concurrent.TimeUnit import nextflow.processor.TaskBean import spock.lang.Specification import spock.lang.Unroll @@ -177,6 +177,158 @@ class SimpleFileCopyStrategyTest extends Specification { } + @Unroll + def 'should copy the right files' () { + + given: + + Path workDir = Files.createTempDirectory('in') + Path outfolder = Files.createTempDirectory('out') + + BashWrapperBuilder bwb = new BashWrapperBuilder([ + script: 'echo Hello World!', + workDir: workDir, + targetDir: outfolder, + scratch: false, + stageOutMode : 'copy', + outputFiles: [source] ] as TaskBean) + + Path scriptPath = bwb.build() + + inputffiles.forEach { + String p = workDir.toAbsolutePath().toString() + //create subdirectories, if needed + if( it.contains("/") ) { + println("path: $p/${it.substring(0, it.lastIndexOf('/'))}") + new File( p + "/" + it.substring(0, it.lastIndexOf('/')) ).mkdirs() + } + if( !it.endsWith("/") ) { + new File(p + "/" + it).createNewFile() + } + } + + Process process = [ "bash", scriptPath ].execute(null, workDir.toFile() ) + process.consumeProcessOutput( System.out, System.err ) + process.waitFor(2, TimeUnit.SECONDS ) + + expect: + process.exitValue() == 0 + + for( String r : outputfiles ){ + assert new File(outfolder.toString(), r).exists() + } + for( String r : inputffiles ){ + assert new File(workDir.toString(), r).exists() + } + for( String r : (inputffiles - outputfiles) ){ + assert !new File(outfolder.toString(), r).exists() + } + + cleanup: + workDir?.deleteDir() + outfolder?.deleteDir() + + where: + source | inputffiles | outputfiles + '**.txt' | ['file.txt', 'file2.txt', 'file3.txta', 'a/file.txt', 'a/file3.txta', 'b/file.txt'] | ['file.txt', 'file2.txt', 'a/file.txt', 'b/file.txt'] + '**.foo' | ['file.txt', 'file2.txt', 'file3.txta', 'a/file.txt', 'a/file3.txta', 'b/file.txt'] | [] + '*.txt' | ['file.txt', 'file2.txt', 'file3.txta'] | ['file.txt', 'file2.txt'] + '*' | ['file.txt', 'file2.txt', 'file3.txta', 'a/b.txt'] | ['file.txt', 'file2.txt', 'file3.txta', 'a/b.txt'] + '**' | ['file.txt', 'file2.txt', 'file3.txta', 'a/b.txt'] | ['file.txt', 'file2.txt', 'file3.txta', 'a/b.txt'] + 'a/*' | ['a/file.txt', 'a/file2.txt', 'a/b/a.txt', 'a/d/', 'file.txt'] | ['a/file.txt', 'a/file2.txt', 'a/b/a.txt', 'a/d/'] + 'a/' | ['a/file.txt', 'a/file2.txt', 'a/b/a.txt', 'a/d/', 'file.txt'] | ['a/file.txt', 'a/file2.txt', 'a/b/a.txt', 'a/d/'] + 'a' | ['a/file.txt', 'a/file2.txt', 'a/b/a.txt', 'a/d/', 'file.txt'] | ['a/file.txt', 'a/file2.txt', 'a/b/a.txt', 'a/d/'] + 'a b/*' | ['a b/file.txt', 'a b/file2.txt', 'a b/b/a.txt', 'a b/d/', 'file.txt'] | ['a b/file.txt', 'a b/file2.txt', 'a b/b/a.txt', 'a b/d/'] + 'a b/' | ['a b/file.txt', 'a b/file2.txt', 'a b/b/a.txt', 'a b/d/', 'file.txt'] | ['a b/file.txt', 'a b/file2.txt', 'a b/b/a.txt', 'a b/d/'] + 'a b' | ['a b/file.txt', 'a b/file2.txt', 'a b/b/a.txt', 'a b/d/', 'file.txt'] | ['a b/file.txt', 'a b/file2.txt', 'a b/b/a.txt', 'a b/d/'] + 'a|b/*' | ['a|b/file.txt', 'a|b/file2.txt', 'a|b/b/a.txt', 'a|b/d/', 'file.txt'] | ['a|b/file.txt', 'a|b/file2.txt', 'a|b/b/a.txt', 'a|b/d/'] + 'a/*/c/*' | ['a/b/c/file.txt', 'a/b/c/file2.txt', 'a/b/d/c/file2.txt', 'a/b/file.txt', 'a/file.txt', 'file.txt'] | ['a/b/c/file.txt', 'a/b/c/file2.txt'] + 'a/**/c/*' | ['a/b/c/file.txt', 'a/b/c/file2.txt', 'a/b/d/c/file2.txt', 'a/b/file.txt', 'a/file.txt', 'file.txt'] | ['a/b/c/file.txt', 'a/b/c/file2.txt', 'a/b/d/c/file2.txt'] + 'a/*/c/' | ['a/b/c/file.txt', 'a/b/c/file2.txt', 'a/b/d/c/file2.txt', 'a/b/file.txt', 'a/file.txt', 'file.txt'] | ['a/b/c/file.txt', 'a/b/c/file2.txt'] + 'a/**/c/' | ['a/b/c/file.txt', 'a/b/c/file2.txt', 'a/b/d/c/file2.txt', 'a/b/file.txt', 'a/file.txt', 'file.txt'] | ['a/b/c/file.txt', 'a/b/c/file2.txt', 'a/b/d/c/file2.txt'] + 'a/*/c' | ['a/b/c/file.txt', 'a/b/c/file2.txt', 'a/b/d/c/file2.txt', 'a/b/file.txt', 'a/file.txt', 'file.txt'] | ['a/b/c/file.txt', 'a/b/c/file2.txt'] + 'a/**/c' | ['a/b/c/file.txt', 'a/b/c/file2.txt', 'a/b/d/c/file2.txt', 'a/b/file.txt', 'a/file.txt', 'file.txt'] | ['a/b/c/file.txt', 'a/b/c/file2.txt', 'a/b/d/c/file2.txt'] + 'a/?/c/*' | ['a/b/c/file.txt', 'a/b/c/file2.txt', 'a/b/d/c/file2.txt', 'a/b/file.txt', 'a/file.txt', 'file.txt'] | ['a/b/c/file.txt', 'a/b/c/file2.txt'] + 'a/?/c/' | ['a/b/c/file.txt', 'a/b/c/file2.txt', 'a/b/d/c/file2.txt', 'a/b/file.txt', 'a/file.txt', 'file.txt'] | ['a/b/c/file.txt', 'a/b/c/file2.txt'] + 'a/?/c' | ['a/b/c/file.txt', 'a/b/c/file2.txt', 'a/b/d/c/file2.txt', 'a/b/file.txt', 'a/file.txt', 'file.txt'] | ['a/b/c/file.txt', 'a/b/c/file2.txt'] + '{a,b,c}/*' | ['a/file.txt', 'a/file2.txt', 'b/file.txt', 'b/file2.txt', 'd/file.txt', 'file.txt'] | ['a/file.txt', 'a/file2.txt', 'b/file.txt', 'b/file2.txt'] + '{a,b,c}/' | ['a/file.txt', 'a/file2.txt', 'b/file.txt', 'b/file2.txt', 'd/file.txt', 'file.txt'] | ['a/file.txt', 'a/file2.txt', 'b/file.txt', 'b/file2.txt'] + '{a,b,c}' | ['a/file.txt', 'a/file2.txt', 'b/file.txt', 'b/file2.txt', 'd/file.txt', 'file.txt'] | ['a/file.txt', 'a/file2.txt', 'b/file.txt', 'b/file2.txt'] + '{ab*,b*/*,c}/*' | ['a/file.txt', 'abcd/file2.txt', 'b/file.txt', 'b/file2.txt', 'd/file.txt', 'file.txt'] | ['abcd/file2.txt'] + '{a[ab]c,b*/*}/*' | ['acc/file.txt', 'abc/file2.txt', 'b/file.txt', 'b/file2.txt', 'd/file.txt', 'file.txt'] | ['abc/file2.txt'] + '{a|c*,b*/*}/*' | ['a|c/file.txt', 'abc/file2.txt', 'b/file.txt', 'b/file2.txt', 'd/file.txt', 'file.txt'] | ['a|c/file.txt'] + '[A-Z]/*' | ['A/file.txt', 'a/file2.txt', 'b/file.txt', 'b/file2.txt', 'd/file.txt', 'file.txt'] | ['A/file.txt'] + '[A-Z]/' | ['A/file.txt', 'a/file2.txt', 'b/file.txt', 'b/file2.txt', 'd/file.txt', 'file.txt'] | ['A/file.txt'] + '[A-Z]' | ['A/file.txt', 'a/file2.txt', 'b/file.txt', 'b/file2.txt', 'd/file.txt', 'file.txt'] | ['A/file.txt'] + 'a' | ['A/file.txt', 'a/', 'b/file.txt', 'b/file2.txt', 'd/file.txt', 'file.txt'] | ['a/'] + 'a/b/c' | ['A/file.txt', 'a/b/c/', 'a/b/file.txt', 'b/file2.txt', 'd/file.txt', 'file.txt'] | ['a/b/c/'] + 'abc?\\?.txt' | ['abcde.txt', 'abcd.txt', 'abcd?.txt'] | ['abcd?.txt'] + 'a"b.txt' | ['a\\"b.txt', 'a"b.txt', 'abc.txt'] | ['a"b.txt'] + 'a/**/b/*/d.txt' | ['a/c/d/b/x/d.txt', 'a/c/d/b/x/Y/d.txt'] | ['a/c/d/b/x/d.txt'] + 'a/bc*h/ij*mn/*' | ['a/bcdefgh/ijklmn/a.txt', 'a/bcdfgh/ijklmn/a.txt', 'a/bcdefghi/ijklmn/a.txt'] | ['a/bcdefgh/ijklmn/a.txt', 'a/bcdfgh/ijklmn/a.txt'] + + } + + @Unroll + def 'should copy the right files, multiple outputs' () { + + given: + + Path workDir = Files.createTempDirectory('in') + Path outfolder = Files.createTempDirectory('out') + + BashWrapperBuilder bwb = new BashWrapperBuilder([ + script: 'echo Hello World!', + workDir: workDir, + targetDir: outfolder, + scratch: false, + stageOutMode : 'copy', + outputFiles: source ] as TaskBean) + + Path scriptPath = bwb.build() + + inputffiles.forEach { + String p = workDir.toAbsolutePath().toString() + //create subdirectories, if needed + if( it.contains("/") ) { + println("path: $p/${it.substring(0, it.lastIndexOf('/'))}") + new File( p + "/" + it.substring(0, it.lastIndexOf('/')) ).mkdirs() + } + if( !it.endsWith("/") ) { + new File(p + "/" + it).createNewFile() + } + } + + Process process = [ "bash", scriptPath ].execute(null, workDir.toFile() ) + process.consumeProcessOutput( System.out, System.err ) + process.waitFor(2, TimeUnit.SECONDS ) + + expect: + process.exitValue() == 0 + + for( String r : outputfiles ){ + assert new File(outfolder.toString(), r).exists() + } + for( String r : inputffiles ){ + assert new File(workDir.toString(), r).exists() + } + for( String r : (inputffiles - outputfiles) ){ + assert !new File(outfolder.toString(), r).exists() + } + + cleanup: + workDir?.deleteDir() + outfolder?.deleteDir() + + where: + source | inputffiles | outputfiles + ['*.txt', 'a/*.txt'] | ['file.txt', 'file2.txt', 'file3.txta', 'a/file.txt', 'a/file3.txta', 'b/file.txt'] | ['file.txt', 'file2.txt', 'a/file.txt'] + ['a/*.txt', '*.txt'] | ['file.txt', 'file2.txt', 'file3.txta', 'a/file.txt', 'a/file3.txta', 'b/file.txt'] | ['file.txt', 'file2.txt', 'a/file.txt'] + ['a/*.txt', '**.txt'] | ['file.txt', 'file2.txt', 'file3.txta', 'a/file.txt', 'a/file3.txta', 'b/file.txt'] | ['file.txt', 'file2.txt', 'a/file.txt', 'b/file.txt'] + ['**.txt', 'a/*.txt'] | ['file.txt', 'file2.txt', 'file3.txta', 'a/file.txt', 'a/file3.txta', 'b/file.txt'] | ['file.txt', 'file2.txt', 'a/file.txt', 'b/file.txt'] + + } + def 'should return a valid `mv` command' () { given: @@ -196,6 +348,156 @@ class SimpleFileCopyStrategyTest extends Specification { } + @Unroll + def 'should move the right files' () { + + given: + + Path workDir = Files.createTempDirectory('in') + Path outfolder = Files.createTempDirectory('out') + + BashWrapperBuilder bwb = new BashWrapperBuilder([ + script: 'echo Hello World!', + workDir: workDir, + targetDir: outfolder, + scratch: false, + stageOutMode : 'move', + outputFiles: [source] ] as TaskBean) + + Path scriptPath = bwb.build() + + inputffiles.forEach { + String p = workDir.toAbsolutePath().toString() + //create subdirectories, if needed + if( it.contains("/") ) { + println("path: $p/${it.substring(0, it.lastIndexOf('/'))}") + new File( p + "/" + it.substring(0, it.lastIndexOf('/')) ).mkdirs() + } + if( !it.endsWith("/") ) { + new File(p + "/" + it).createNewFile() + } + } + + Process process = [ "bash", scriptPath ].execute(null, workDir.toFile() ) + process.consumeProcessOutput( System.out, System.err ) + process.waitFor(2, TimeUnit.SECONDS ) + + expect: + process.exitValue() == 0 + + for( String r : outputfiles ){ + assert new File(outfolder.toString(), r).exists() + assert !new File(workDir.toString(), r).exists() + } + for( String r : (inputffiles - outputfiles) ){ + assert !new File(outfolder.toString(), r).exists() + assert new File(workDir.toString(), r).exists() + } + + cleanup: + workDir?.deleteDir() + outfolder?.deleteDir() + + where: + source | inputffiles | outputfiles + '**.txt' | ['file.txt', 'file2.txt', 'file3.txta', 'a/file.txt', 'a/file3.txta', 'b/file.txt'] | ['file.txt', 'file2.txt', 'a/file.txt', 'b/file.txt'] + '**.foo' | ['file.txt', 'file2.txt', 'file3.txta', 'a/file.txt', 'a/file3.txta', 'b/file.txt'] | [] + '*.txt' | ['file.txt', 'file2.txt', 'file3.txta'] | ['file.txt', 'file2.txt'] + '*' | ['file.txt', 'file2.txt', 'file3.txta', 'a/b.txt'] | ['file.txt', 'file2.txt', 'file3.txta', 'a/b.txt'] + '**' | ['file.txt', 'file2.txt', 'file3.txta', 'a/b.txt'] | ['file.txt', 'file2.txt', 'file3.txta', 'a/b.txt'] + 'a/*' | ['a/file.txt', 'a/file2.txt', 'a/b/a.txt', 'a/d/', 'file.txt'] | ['a/file.txt', 'a/file2.txt', 'a/b/a.txt', 'a/d/'] + 'a/' | ['a/file.txt', 'a/file2.txt', 'a/b/a.txt', 'a/d/', 'file.txt'] | ['a/file.txt', 'a/file2.txt', 'a/b/a.txt', 'a/d/'] + 'a' | ['a/file.txt', 'a/file2.txt', 'a/b/a.txt', 'a/d/', 'file.txt'] | ['a/file.txt', 'a/file2.txt', 'a/b/a.txt', 'a/d/'] + 'a b/*' | ['a b/file.txt', 'a b/file2.txt', 'a b/b/a.txt', 'a b/d/', 'file.txt'] | ['a b/file.txt', 'a b/file2.txt', 'a b/b/a.txt', 'a b/d/'] + 'a b/' | ['a b/file.txt', 'a b/file2.txt', 'a b/b/a.txt', 'a b/d/', 'file.txt'] | ['a b/file.txt', 'a b/file2.txt', 'a b/b/a.txt', 'a b/d/'] + 'a b' | ['a b/file.txt', 'a b/file2.txt', 'a b/b/a.txt', 'a b/d/', 'file.txt'] | ['a b/file.txt', 'a b/file2.txt', 'a b/b/a.txt', 'a b/d/'] + 'a|b/*' | ['a|b/file.txt', 'a|b/file2.txt', 'a|b/b/a.txt', 'a|b/d/', 'file.txt'] | ['a|b/file.txt', 'a|b/file2.txt', 'a|b/b/a.txt', 'a|b/d/'] + 'a/*/c/*' | ['a/b/c/file.txt', 'a/b/c/file2.txt', 'a/b/d/c/file2.txt', 'a/b/file.txt', 'a/file.txt', 'file.txt'] | ['a/b/c/file.txt', 'a/b/c/file2.txt'] + 'a/**/c/*' | ['a/b/c/file.txt', 'a/b/c/file2.txt', 'a/b/d/c/file2.txt', 'a/b/file.txt', 'a/file.txt', 'file.txt'] | ['a/b/c/file.txt', 'a/b/c/file2.txt', 'a/b/d/c/file2.txt'] + 'a/*/c/' | ['a/b/c/file.txt', 'a/b/c/file2.txt', 'a/b/d/c/file2.txt', 'a/b/file.txt', 'a/file.txt', 'file.txt'] | ['a/b/c/file.txt', 'a/b/c/file2.txt'] + 'a/**/c/' | ['a/b/c/file.txt', 'a/b/c/file2.txt', 'a/b/d/c/file2.txt', 'a/b/file.txt', 'a/file.txt', 'file.txt'] | ['a/b/c/file.txt', 'a/b/c/file2.txt', 'a/b/d/c/file2.txt'] + 'a/*/c' | ['a/b/c/file.txt', 'a/b/c/file2.txt', 'a/b/d/c/file2.txt', 'a/b/file.txt', 'a/file.txt', 'file.txt'] | ['a/b/c/file.txt', 'a/b/c/file2.txt'] + 'a/**/c' | ['a/b/c/file.txt', 'a/b/c/file2.txt', 'a/b/d/c/file2.txt', 'a/b/file.txt', 'a/file.txt', 'file.txt'] | ['a/b/c/file.txt', 'a/b/c/file2.txt', 'a/b/d/c/file2.txt'] + 'a/?/c/*' | ['a/b/c/file.txt', 'a/b/c/file2.txt', 'a/b/d/c/file2.txt', 'a/b/file.txt', 'a/file.txt', 'file.txt'] | ['a/b/c/file.txt', 'a/b/c/file2.txt'] + 'a/?/c/' | ['a/b/c/file.txt', 'a/b/c/file2.txt', 'a/b/d/c/file2.txt', 'a/b/file.txt', 'a/file.txt', 'file.txt'] | ['a/b/c/file.txt', 'a/b/c/file2.txt'] + 'a/?/c' | ['a/b/c/file.txt', 'a/b/c/file2.txt', 'a/b/d/c/file2.txt', 'a/b/file.txt', 'a/file.txt', 'file.txt'] | ['a/b/c/file.txt', 'a/b/c/file2.txt'] + '{a,b,c}/*' | ['a/file.txt', 'a/file2.txt', 'b/file.txt', 'b/file2.txt', 'd/file.txt', 'file.txt'] | ['a/file.txt', 'a/file2.txt', 'b/file.txt', 'b/file2.txt'] + '{a,b,c}/' | ['a/file.txt', 'a/file2.txt', 'b/file.txt', 'b/file2.txt', 'd/file.txt', 'file.txt'] | ['a/file.txt', 'a/file2.txt', 'b/file.txt', 'b/file2.txt'] + '{a,b,c}' | ['a/file.txt', 'a/file2.txt', 'b/file.txt', 'b/file2.txt', 'd/file.txt', 'file.txt'] | ['a/file.txt', 'a/file2.txt', 'b/file.txt', 'b/file2.txt'] + '{ab*,b*/*,c}/*' | ['a/file.txt', 'abcd/file2.txt', 'b/file.txt', 'b/file2.txt', 'd/file.txt', 'file.txt'] | ['abcd/file2.txt'] + '{a[ab]c,b*/*}/*' | ['acc/file.txt', 'abc/file2.txt', 'b/file.txt', 'b/file2.txt', 'd/file.txt', 'file.txt'] | ['abc/file2.txt'] + '{a|c*,b*/*}/*' | ['a|c/file.txt', 'abc/file2.txt', 'b/file.txt', 'b/file2.txt', 'd/file.txt', 'file.txt'] | ['a|c/file.txt'] + '[A-Z]/*' | ['A/file.txt', 'a/file2.txt', 'b/file.txt', 'b/file2.txt', 'd/file.txt', 'file.txt'] | ['A/file.txt'] + '[A-Z]/' | ['A/file.txt', 'a/file2.txt', 'b/file.txt', 'b/file2.txt', 'd/file.txt', 'file.txt'] | ['A/file.txt'] + '[A-Z]' | ['A/file.txt', 'a/file2.txt', 'b/file.txt', 'b/file2.txt', 'd/file.txt', 'file.txt'] | ['A/file.txt'] + 'a' | ['A/file.txt', 'a/', 'b/file.txt', 'b/file2.txt', 'd/file.txt', 'file.txt'] | ['a/'] + 'a/b/c' | ['A/file.txt', 'a/b/c/', 'a/b/file.txt', 'b/file2.txt', 'd/file.txt', 'file.txt'] | ['a/b/c/'] + 'abc?\\?.txt' | ['abcde.txt', 'abcd.txt', 'abcd?.txt'] | ['abcd?.txt'] + 'a"b.txt' | ['a\\"b.txt', 'a"b.txt', 'abc.txt'] | ['a"b.txt'] + 'a/**/b/*/d.txt' | ['a/c/d/b/x/d.txt', 'a/c/d/b/x/Y/d.txt'] | ['a/c/d/b/x/d.txt'] + 'a/bc*h/ij*mn/*' | ['a/bcdefgh/ijklmn/a.txt', 'a/bcdfgh/ijklmn/a.txt', 'a/bcdefghi/ijklmn/a.txt'] | ['a/bcdefgh/ijklmn/a.txt', 'a/bcdfgh/ijklmn/a.txt'] + + } + + @Unroll + def 'should move the right files, multiple outputs' () { + + given: + + Path workDir = Files.createTempDirectory('in') + Path outfolder = Files.createTempDirectory('out') + + BashWrapperBuilder bwb = new BashWrapperBuilder([ + script: 'echo Hello World!', + workDir: workDir, + targetDir: outfolder, + scratch: false, + stageOutMode : 'move', + outputFiles: source ] as TaskBean) + + Path scriptPath = bwb.build() + + inputffiles.forEach { + String p = workDir.toAbsolutePath().toString() + //create subdirectories, if needed + if( it.contains("/") ) { + println("path: $p/${it.substring(0, it.lastIndexOf('/'))}") + new File( p + "/" + it.substring(0, it.lastIndexOf('/')) ).mkdirs() + } + if( !it.endsWith("/") ) { + new File(p + "/" + it).createNewFile() + } + } + + Process process = [ "bash", scriptPath ].execute(null, workDir.toFile() ) + process.consumeProcessOutput( System.out, System.err ) + process.waitFor(2, TimeUnit.SECONDS ) + + expect: + process.exitValue() == 0 + + for( String r : outputfiles ){ + assert new File(outfolder.toString(), r).exists() + assert !new File(workDir.toString(), r).exists() + } + for( String r : (inputffiles - outputfiles) ){ + assert !new File(outfolder.toString(), r).exists() + assert new File(workDir.toString(), r).exists() + } + + cleanup: + workDir?.deleteDir() + outfolder?.deleteDir() + + where: + source | inputffiles | outputfiles + ['*.txt', 'a/*.txt'] | ['file.txt', 'file2.txt', 'file3.txta', 'a/file.txt', 'a/file3.txta', 'b/file.txt'] | ['file.txt', 'file2.txt', 'a/file.txt'] + ['a/*.txt', '*.txt'] | ['file.txt', 'file2.txt', 'file3.txta', 'a/file.txt', 'a/file3.txta', 'b/file.txt'] | ['file.txt', 'file2.txt', 'a/file.txt'] + ['a/*.txt', '**.txt'] | ['file.txt', 'file2.txt', 'file3.txta', 'a/file.txt', 'a/file3.txta', 'b/file.txt'] | ['file.txt', 'file2.txt', 'a/file.txt', 'b/file.txt'] + ['**.txt', 'a/*.txt'] | ['file.txt', 'file2.txt', 'file3.txta', 'a/file.txt', 'a/file3.txta', 'b/file.txt'] | ['file.txt', 'file2.txt', 'a/file.txt', 'b/file.txt'] + + } + def 'should return a valid `rsync` command' () { given: @@ -270,9 +572,12 @@ class SimpleFileCopyStrategyTest extends Specification { then: script == ''' IFS=$'\\n' - for name in $(eval "ls -1d simple.txt my/path/file.bam" | sort | uniq); do + pathes=`ls -1d simple.txt my/path/file.bam | sort | uniq` + set -f + for name in $pathes; do nxf_fs_copy "$name" /target/work\\ dir || true done + set +f unset IFS ''' .stripIndent().trim() @@ -293,9 +598,12 @@ class SimpleFileCopyStrategyTest extends Specification { then: script == ''' IFS=$'\\n' - for name in $(eval "ls -1d simple.txt my/path/file.bam" | sort | uniq); do + pathes=`ls -1d simple.txt my/path/file.bam | sort | uniq` + set -f + for name in $pathes; do nxf_fs_move "$name" /target/store || true done + set +f unset IFS ''' .stripIndent().trim() @@ -315,9 +623,12 @@ class SimpleFileCopyStrategyTest extends Specification { then: script == ''' IFS=$'\\n' - for name in $(eval "ls -1d simple.txt my/path/file.bam" | sort | uniq); do + pathes=`ls -1d simple.txt my/path/file.bam | sort | uniq` + set -f + for name in $pathes; do nxf_fs_rsync "$name" /target/work\\'s || true done + set +f unset IFS ''' .stripIndent().trim() diff --git a/modules/nf-commons/src/main/nextflow/util/Escape.groovy b/modules/nf-commons/src/main/nextflow/util/Escape.groovy index a66f4983f6..111fe003d5 100644 --- a/modules/nf-commons/src/main/nextflow/util/Escape.groovy +++ b/modules/nf-commons/src/main/nextflow/util/Escape.groovy @@ -55,7 +55,11 @@ class Escape { replace(WILDCARDS, str) } - static String path(String val) { + static String path(String val, boolean glob = false) { + List SPECIAL_CHARS = Escape.SPECIAL_CHARS + if ( glob ){ + SPECIAL_CHARS = SPECIAL_CHARS.findAll {it != '\\' } + } replace(SPECIAL_CHARS, val, true) } diff --git a/modules/nf-commons/src/test/nextflow/util/EscapeTest.groovy b/modules/nf-commons/src/test/nextflow/util/EscapeTest.groovy index df851a0d51..abe9f6ece9 100644 --- a/modules/nf-commons/src/test/nextflow/util/EscapeTest.groovy +++ b/modules/nf-commons/src/test/nextflow/util/EscapeTest.groovy @@ -70,6 +70,16 @@ class EscapeTest extends Specification { Escape.wildcards('file_[!a].txt') == 'file_\\[\\!a\\].txt' } + def 'should ignore globs in file names' () { + + expect: + Escape.path('hell?.txt', true ) == "hell?.txt" + Escape.path('hell\\?.txt', true ) == "hell\\?.txt" + Escape.path('hell*.txt', true ) == "hell*.txt" + Escape.path('hell**.txt', true ) == "hell**.txt" + + } + def 'should escape cli' () { expect: Escape.cli('nextflow','run','this') == 'nextflow run this' diff --git a/plugins/nf-amazon/src/test/nextflow/executor/BashWrapperBuilderWithS3Test.groovy b/plugins/nf-amazon/src/test/nextflow/executor/BashWrapperBuilderWithS3Test.groovy index e4609ac45d..0d71d9cff1 100644 --- a/plugins/nf-amazon/src/test/nextflow/executor/BashWrapperBuilderWithS3Test.groovy +++ b/plugins/nf-amazon/src/test/nextflow/executor/BashWrapperBuilderWithS3Test.groovy @@ -58,9 +58,12 @@ class BashWrapperBuilderWithS3Test extends Specification { then: binding.unstage_outputs == '''\ IFS=$'\\n' - for name in $(eval "ls -1d test.bam test.bai" | sort | uniq); do + pathes=`ls -1d test.bam test.bai | sort | uniq` + set -f + for name in $pathes; do nxf_s3_upload '$name' s3://some/bucket || true done + set +f unset IFS '''.stripIndent().rightTrim() diff --git a/plugins/nf-azure/src/test/nextflow/executor/BashWrapperBuilderWithAzTest.groovy b/plugins/nf-azure/src/test/nextflow/executor/BashWrapperBuilderWithAzTest.groovy index db14dfd18a..3d35b6f27f 100644 --- a/plugins/nf-azure/src/test/nextflow/executor/BashWrapperBuilderWithAzTest.groovy +++ b/plugins/nf-azure/src/test/nextflow/executor/BashWrapperBuilderWithAzTest.groovy @@ -46,9 +46,12 @@ class BashWrapperBuilderWithAzTest extends Specification { then: binding.unstage_outputs == """\ IFS=\$'\\n' - for name in \$(eval "ls -1d test.bam test.bai" | sort | uniq); do + pathes=`ls -1d test.bam test.bai | sort | uniq` + set -f + for name in \$pathes; do nxf_az_upload '\$name' '${AzHelper.toHttpUrl(target)}' || true done + set +f unset IFS """.stripIndent().rightTrim()