Skip to content

Commit

Permalink
Improve dsl detection [ci fast]
Browse files Browse the repository at this point in the history 
This commit tries to determine the use of DSL1 if the explict
setting 'nextflow.enable.dsl' has not been provided in the
nextflow config or script file.

When the use of DSL1 cannot be inferred, it looks for the
DSL mode specified via the env variable NXF_DEFAULT_DSL.

If the env variable is found it finally defaults to DSL2.

Signed-off-by: Paolo Di Tommaso <[email protected]>
  • Loading branch information
@pditommaso
pditommaso committed May 15, 2022
1 parent da101e8 commit 1f836f8
Show file tree
Hide file tree
Showing 8 changed files with 260 additions and 59 deletions.
40 changes: 28 additions & 12 deletions modules/nextflow/src/main/groovy/nextflow/NextflowMeta.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,11 @@ package nextflow
import java.text.SimpleDateFormat
import java.util.regex.Pattern

import groovy.transform.CompileStatic
import groovy.transform.EqualsAndHashCode
import groovy.transform.ToString
import groovy.util.logging.Slf4j
import nextflow.exception.AbortOperationException
import nextflow.util.VersionNumber
import static nextflow.extension.Bolts.DATETIME_FORMAT

Expand All @@ -14,13 +16,18 @@ import static nextflow.extension.Bolts.DATETIME_FORMAT
*
* @author Paolo Di Tommaso <[email protected]>
*/
@CompileStatic
@Singleton(strict = false)
@ToString(includeNames = true)
@EqualsAndHashCode
class NextflowMeta {

private static final Pattern DSL_DECLARATION = ~/(?m)^\s*(nextflow\.(preview|enable)\.dsl\s*=\s*(\d))\s*(;?\s*)?(;?\/{2}.*)?$/

private static final Pattern DSL1_INPUT = ~/(?m)input:\s*(tuple|file|path|val|env|stdin)\b.*\s.*\bfrom\b.+$/
private static final Pattern DSL1_OUTPUT = ~/(?m)output:\s*(tuple|file|path|val|env|stdout)\b.*\s.*\binto\b.+$/
private static final Pattern DSL2_WORKFLOW = ~/\s+workflow\b.*\s*\{(\n|\r|.)*}/

private static boolean ignoreWarnDsl2 = System.getenv('NXF_IGNORE_WARN_DSL2')=='true'

static trait Flags {
Expand Down Expand Up @@ -114,7 +121,7 @@ class NextflowMeta {
* {@code true} when the workflow script uses DSL2 syntax, {@code false} otherwise.
*/
boolean isDsl2() {
enable.dsl == 2
enable.dsl == 2f
}

/**
Expand All @@ -125,15 +132,22 @@ class NextflowMeta {
*/
@Deprecated
boolean isDsl2Final() {
enable.dsl == 2
enable.dsl == 2f
}

void enableDsl2() {
this.enable.dsl = 2
this.enable.dsl = 2f
}

void disableDsl2() {
enable.dsl = 1
enable.dsl = 1f
}

void enableDsl(String value) {
if( value !in ['1','2'] ) {
throw new AbortOperationException("Invalid Nextflow DSL value: $value")
}
this.enable.dsl = value=='1' ? 1f : 2f
}

boolean isStrictModeEnabled() {
Expand All @@ -144,23 +158,25 @@ class NextflowMeta {
enable.strict = mode
}

void checkDsl2Mode(String script) {
static String checkDslMode(String script) {
final matcher = DSL_DECLARATION.matcher(script)
final mode = matcher.find() ? matcher.group(2) : null
if( !mode )
return
return null
final ver = matcher.group(3)
if( mode == 'enable' ) {
if( ver=='2' )
enableDsl2()
else if( ver=='1' )
disableDsl2()
else
throw new IllegalArgumentException("Unknown nextflow DSL version: ${ver}")
return ver
}
else if( mode == 'preview' )
throw new IllegalArgumentException("Preview nextflow mode 'preview' is not supported anymore -- Please use `nextflow.enable.dsl=2` instead")
else
throw new IllegalArgumentException("Unknown nextflow mode=${matcher.group(1)}")
}

static boolean probeDls1(String script) {
boolean hasDsl1Input = DSL1_INPUT.matcher(script).find()
boolean hasDsl1Output = DSL1_OUTPUT.matcher(script).find()
boolean hasWorkflowDef = DSL2_WORKFLOW.matcher(script).find()
return (hasDsl1Input || hasDsl1Output) && !hasWorkflowDef
}
}
59 changes: 41 additions & 18 deletions modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,12 @@ import org.yaml.snakeyaml.Yaml
@Parameters(commandDescription = "Execute a pipeline project")
class CmdRun extends CmdBase implements HubOptions {

static final Pattern RUN_NAME_PATTERN = Pattern.compile(/^[a-z](?:[a-z\d]|[-_](?=[a-z\d])){0,79}$/, Pattern.CASE_INSENSITIVE)
static final public Pattern RUN_NAME_PATTERN = Pattern.compile(/^[a-z](?:[a-z\d]|[-_](?=[a-z\d])){0,79}$/, Pattern.CASE_INSENSITIVE)

static List<String> VALID_PARAMS_FILE = ['json', 'yml', 'yaml']
static final public List<String> VALID_PARAMS_FILE = ['json', 'yml', 'yaml']

static final public DSL2 = '2'
static final public DSL1 = '1'

static {
// install the custom pool factory for GPars threads
Expand Down Expand Up @@ -342,22 +345,10 @@ class CmdRun extends CmdBase implements HubOptions {
log.debug "Enabling nextflow strict mode"
NextflowMeta.instance.strictMode(true)
}
// -- try determine DSL version from config file
final DSL2 = '2'
final DSL1 = '1'
final defaultDsl = sysEnv.get('NXF_DEFAULT_DSL') ?: DSL2
final dsl = config.navigate('nextflow.enable.dsl', defaultDsl) as String
if( dsl=='2' )
NextflowMeta.instance.enableDsl2()
else if( dsl=='1' )
NextflowMeta.instance.disableDsl2()
else
throw new AbortOperationException("Invalid Nextflow DSL value: $dsl")

// -- script can still override the DSL version
NextflowMeta.instance.checkDsl2Mode(scriptFile.main.text)

// -- show launch info
// -- determine dsl mode
final dsl = detectDslMode(config, scriptFile.main.text, sysEnv)
NextflowMeta.instance.enableDsl(dsl)
// -- show launch info
final ver = NF.dsl2 ? DSL2 : DSL1
if( scriptFile.repository )
log.info "Launching `$scriptFile.repository` [$runName] DSL${ver} - revision: ${scriptFile.revisionInfo}"
Expand All @@ -366,6 +357,38 @@ class CmdRun extends CmdBase implements HubOptions {

}

static String detectDslMode(ConfigMap config, String scriptText, Map sysEnv) {
// -- try determine DSL version from config file

final dsl = config.navigate('nextflow.enable.dsl') as String

// -- script can still override the DSL version
final scriptDsl = NextflowMeta.checkDslMode(scriptText)
if( scriptDsl ) {
log.debug("Applied DSL=$scriptDsl from script declararion")
return scriptDsl
}
else if( dsl ) {
log.debug("Applied DSL=$scriptDsl from config declaration")
return dsl
}
// -- if still unknown try probing for DSL1
if( NextflowMeta.probeDls1(scriptText) ) {
log.debug "Applied DSL=1 by probing script field"
return DSL1
}

final envDsl = sysEnv.get('NXF_DEFAULT_DSL')
if( envDsl ) {
log.debug "Applied DSL=$envDsl from NXF_DEFAULT_DSL variable"
return envDsl
}
else {
log.debug "Applied DSL=2 by global default"
return DSL2
}
}

protected void checkRunName() {
if( runName == 'last' )
throw new AbortOperationException("Not a valid run name: `last`")
Expand Down
146 changes: 125 additions & 21 deletions modules/nextflow/src/test/groovy/nextflow/NextflowMetaTest.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,10 @@ class NextflowMetaTest extends Specification {
def tz = TimeZone.getTimeZone('UTC')
def fmt = new SimpleDateFormat(DATETIME_FORMAT)
fmt.setTimeZone(tz)
fmt.format(date) + ' ' + tz.getDisplayName( true, TimeZone.SHORT )
fmt.format(date) + ' ' + tz.getDisplayName(true, TimeZone.SHORT)
}

def 'should convert to map' () {
def 'should convert to map'() {
given:
def meta = new NextflowMeta('10.12.0', 123, Const.APP_TIMESTAMP_UTC)
meta.enableDsl2()
Expand All @@ -32,34 +32,138 @@ class NextflowMetaTest extends Specification {
map.version == '10.12.0'
map.build == 123
map.enable.dsl == 2
dateToString((Date)map.timestamp) == Const.APP_TIMESTAMP_UTC
dateToString((Date) map.timestamp) == Const.APP_TIMESTAMP_UTC

}

@Unroll
def 'should find dsl2 declaration: #SCRIPT' () {
given:
def meta = new NextflowMeta()

def 'should find dsl2 declaration: #SCRIPT'() {
when:
meta.checkDsl2Mode(SCRIPT)
def result = NextflowMeta.checkDslMode(SCRIPT)
then:
meta.isDsl2() == DSL2
meta.isDsl2Final() == FINAL
result == RESULT

where:
DSL2 | FINAL | SCRIPT
false | false | 'hello'
false | false | 'nextflow.enable.dsl=1'
RESULT | SCRIPT
null | 'hello'
'1' | 'nextflow.enable.dsl=1'
and:
'2' | 'nextflow.enable.dsl=2'
'2' | 'nextflow.enable.dsl = 2'
'2' | 'nextflow.enable.dsl = 2;'
'2' | '#!/bin/env nextflow\nnextflow.enable.dsl=2\nprintln foo'
'2' | 'nextflow.enable.dsl = 2 // a comment'
'2' | 'nextflow.enable.dsl = 2;// another comment'
'2' | 'nextflow.enable.dsl=2 // another comment'
and:
'1' | 'nextflow.enable.dsl = 1'
}

def 'should probe dsl1' () {

expect:
NextflowMeta.probeDls1('''
process foo {
input:
file transcriptome from transcriptome_file
}
''')
and:
NextflowMeta.probeDls1('''
process foo {
input:
file transcriptome
from transcriptome_file
}
''')

NextflowMeta.probeDls1('''
process foo {
input: file transcriptome from transcriptome_file
}
''')


and:
true | true | 'nextflow.enable.dsl=2'
true | true | 'nextflow.enable.dsl = 2'
true | true | 'nextflow.enable.dsl = 2;'
true | true | '#!/bin/env nextflow\nnextflow.enable.dsl=2\nprintln foo'
true | true | 'nextflow.enable.dsl = 2 // a comment'
true | true | 'nextflow.enable.dsl = 2;// another comment'
true | true | 'nextflow.enable.dsl=2 // another comment'
NextflowMeta.probeDls1('''
process foo {
output:
file transcriptome into transcriptome_file
}
''')
and:
NextflowMeta.probeDls1('''
process foo {
output:
file transcriptome
into transcriptome_file
}
''')

NextflowMeta.probeDls1('''
process foo {
output: file transcriptome into transcriptome_file
}
''')

and:
NextflowMeta.probeDls1('''
process foo {
input:
file transcriptome from transcriptome_file
output:
file transcriptome into transcriptome_file
}
''')

and:
!NextflowMeta.probeDls1('''
process foo {
input:
file transcriptome
output:
file transcriptome
}
''')

and:
!NextflowMeta.probeDls1('''
process foo {
input:
file transcriptome from transcriptome_file
output:
file transcriptome into transcriptome_file
}
workflow {
foo()
}
''')

and:
!NextflowMeta.probeDls1('''
process foo {
input:
file transcriptome from transcriptome_file
output:
file transcriptome into transcriptome_file
}
workflow{ foo() }
''')

and:
false | false | 'nextflow.enable.dsl = 1'
!NextflowMeta.probeDls1('''
process foo {
input:
file transcriptome from transcriptome_file
output:
file transcriptome into transcriptome_file
}
workflow bar {
foo()
}
''')
}
}
Loading

0 comments on commit 1f836f8

Please sign in to comment.