From 5d00fe744971fb16b542b45d350c0d8ffc957187 Mon Sep 17 00:00:00 2001
From: Samuel Lee <lee.samuel.k@gmail.com>
Date: Mon, 4 Nov 2019 15:38:35 -0500
Subject: [PATCH] Added ability to override THEANO_FLAGS environment variable
 in gCNV tools.

---
 .../DetermineGermlineContigPloidy.java        |  8 ++++++++
 .../tools/copynumber/GermlineCNVCaller.java   |  8 ++++++++
 .../PostprocessGermlineCNVCalls.java          | 19 ++++++++++++++++++-
 .../copynumber/case_denoising_calling.py      |  9 +++++++--
 .../case_determine_ploidy_and_depth.py        | 12 ++++++++++--
 .../copynumber/cohort_denoising_calling.py    | 11 ++++++++---
 .../cohort_determine_ploidy_and_depth.py      | 12 ++++++++++--
 .../tools/copynumber/segment_gcnv_calls.py    |  9 +++++++--
 8 files changed, 76 insertions(+), 12 deletions(-)

diff --git a/src/main/java/org/broadinstitute/hellbender/tools/copynumber/DetermineGermlineContigPloidy.java b/src/main/java/org/broadinstitute/hellbender/tools/copynumber/DetermineGermlineContigPloidy.java
index c29d1067ef8..fe07f4619be 100644
--- a/src/main/java/org/broadinstitute/hellbender/tools/copynumber/DetermineGermlineContigPloidy.java
+++ b/src/main/java/org/broadinstitute/hellbender/tools/copynumber/DetermineGermlineContigPloidy.java
@@ -58,6 +58,14 @@
  * the python environment is already set up. Otherwise, the environment must be created and activated as described in the
  * main GATK README.md file.</p>
  *
+ * <p>Advanced users may wish to set the <code>THEANO_FLAGS</code> environment variable to override the GATK theano
+ * configuration. For example, by running
+ * <code>THEANO_FLAGS="base_compiledir=PATH/TO/BASE_COMPILEDIR" gatk DetermineGermlineContigPloidy ...</code>, users can specify
+ * the theano compilation directory (which is set to <code>$HOME/.theano</code> by default).  See theano documentation
+ * at <a href="http://deeplearning.net/software/theano/library/config.html">
+ *     http://deeplearning.net/software/theano/library/config.html</a>.
+ * </p>
+ *
  * <h3>Tool run modes</h3>
  *
  * <p>This tool has two operation modes as described below:</p>
diff --git a/src/main/java/org/broadinstitute/hellbender/tools/copynumber/GermlineCNVCaller.java b/src/main/java/org/broadinstitute/hellbender/tools/copynumber/GermlineCNVCaller.java
index 8875ee448b1..99dfa000c72 100644
--- a/src/main/java/org/broadinstitute/hellbender/tools/copynumber/GermlineCNVCaller.java
+++ b/src/main/java/org/broadinstitute/hellbender/tools/copynumber/GermlineCNVCaller.java
@@ -70,6 +70,14 @@
  * the python environment is already set up. Otherwise, the environment must be created and activated as described in the
  * main GATK README.md file.</p>
  *
+ * <p>Advanced users may wish to set the <code>THEANO_FLAGS</code> environment variable to override the GATK theano
+ * configuration. For example, by running
+ * <code>THEANO_FLAGS="base_compiledir=PATH/TO/BASE_COMPILEDIR" gatk GermlineCNVCaller ...</code>, users can specify
+ * the theano compilation directory (which is set to <code>$HOME/.theano</code> by default).  See theano documentation
+ * at <a href="http://deeplearning.net/software/theano/library/config.html">
+ *     http://deeplearning.net/software/theano/library/config.html</a>.
+ * </p>
+ *
  * <h3>Tool run modes</h3>
  * <dl>
  *     <dt>COHORT mode:</dt>
diff --git a/src/main/java/org/broadinstitute/hellbender/tools/copynumber/PostprocessGermlineCNVCalls.java b/src/main/java/org/broadinstitute/hellbender/tools/copynumber/PostprocessGermlineCNVCalls.java
index e7051457866..73eb2200351 100644
--- a/src/main/java/org/broadinstitute/hellbender/tools/copynumber/PostprocessGermlineCNVCalls.java
+++ b/src/main/java/org/broadinstitute/hellbender/tools/copynumber/PostprocessGermlineCNVCalls.java
@@ -61,7 +61,24 @@
  * calls of {@link DetermineGermlineContigPloidy}.</p>
  *
  * <p>Finally, the tool concatenates posterior means for denoised copy ratios from all the call shards produced by
- * the {@link GermlineCNVCaller} into a single file. </p>
+ * the {@link GermlineCNVCaller} into a single file.</p>
+ *
+ * <h3>Python environment setup</h3>
+ *
+ * <p>The computation done by this tool, aside from input data parsing and validation, is performed outside of the Java
+ * Virtual Machine and using the <em>gCNV computational python module</em>, namely {@code gcnvkernel}. It is crucial that
+ * the user has properly set up a python conda environment with {@code gcnvkernel} and its dependencies
+ * installed. If the user intends to run {@link PostprocessGermlineCNVCalls} using one of the official GATK Docker images,
+ * the python environment is already set up. Otherwise, the environment must be created and activated as described in the
+ * main GATK README.md file.</p>
+ *
+ * <p>Advanced users may wish to set the <code>THEANO_FLAGS</code> environment variable to override the GATK theano
+ * configuration. For example, by running
+ * <code>THEANO_FLAGS="base_compiledir=PATH/TO/BASE_COMPILEDIR" gatk PostprocessGermlineCNVCalls ...</code>, users can specify
+ * the theano compilation directory (which is set to <code>$HOME/.theano</code> by default).  See theano documentation
+ * at <a href="http://deeplearning.net/software/theano/library/config.html">
+ *     http://deeplearning.net/software/theano/library/config.html</a>.
+ * </p>
  *
  * <h3>Required inputs:</h3>
  * <ul>
diff --git a/src/main/resources/org/broadinstitute/hellbender/tools/copynumber/case_denoising_calling.py b/src/main/resources/org/broadinstitute/hellbender/tools/copynumber/case_denoising_calling.py
index 54cbd2a834e..0fe7b4f6ce2 100644
--- a/src/main/resources/org/broadinstitute/hellbender/tools/copynumber/case_denoising_calling.py
+++ b/src/main/resources/org/broadinstitute/hellbender/tools/copynumber/case_denoising_calling.py
@@ -1,8 +1,11 @@
 import os
 
 # set theano flags
-os.environ["THEANO_FLAGS"] = "device=cpu,floatX=float64,optimizer=fast_run,compute_test_value=ignore," + \
-                             "openmp=true,blas.ldflags=-lmkl_rt,openmp_elemwise_minsize=10"
+user_theano_flags = os.environ.get("THEANO_FLAGS")
+default_theano_flags = "device=cpu,floatX=float64,optimizer=fast_run,compute_test_value=ignore," + \
+                       "openmp=true,blas.ldflags=-lmkl_rt,openmp_elemwise_minsize=10"
+theano_flags = default_theano_flags + ("" if user_theano_flags is None else "," + user_theano_flags)
+os.environ["THEANO_FLAGS"] = theano_flags
 
 import logging
 import argparse
@@ -148,6 +151,8 @@ def update_args_dict_from_saved_model(input_model_path: str,
     args = parser.parse_args()
     gcnvkernel.cli_commons.set_logging_config_from_args(args)
 
+    logger.info("THEANO_FLAGS environment variable has been set to: {theano_flags}".format(theano_flags=theano_flags))
+
     # check gcnvkernel version in the input model path
     gcnvkernel.io_commons.check_gcnvkernel_version_from_path(args.input_model_path)
 
diff --git a/src/main/resources/org/broadinstitute/hellbender/tools/copynumber/case_determine_ploidy_and_depth.py b/src/main/resources/org/broadinstitute/hellbender/tools/copynumber/case_determine_ploidy_and_depth.py
index 0929dbf2048..1ffa372d5f7 100644
--- a/src/main/resources/org/broadinstitute/hellbender/tools/copynumber/case_determine_ploidy_and_depth.py
+++ b/src/main/resources/org/broadinstitute/hellbender/tools/copynumber/case_determine_ploidy_and_depth.py
@@ -1,12 +1,18 @@
 import os
 
 # set theano flags
-os.environ["THEANO_FLAGS"] = "device=cpu,floatX=float64,optimizer=fast_run,compute_test_value=ignore," + \
-                             "openmp=true,blas.ldflags=-lmkl_rt,openmp_elemwise_minsize=10"
+user_theano_flags = os.environ.get("THEANO_FLAGS")
+default_theano_flags = "device=cpu,floatX=float64,optimizer=fast_run,compute_test_value=ignore," + \
+                       "openmp=true,blas.ldflags=-lmkl_rt,openmp_elemwise_minsize=10"
+theano_flags = default_theano_flags + ("" if user_theano_flags is None else "," + user_theano_flags)
+os.environ["THEANO_FLAGS"] = theano_flags
 
+import logging
 import argparse
 import gcnvkernel
 
+logger = logging.getLogger("case_determine_ploidy_and_depth")
+
 parser = argparse.ArgumentParser(description="gCNV contig ploidy and read depth determination tool",
                                  formatter_class=gcnvkernel.cli_commons.GCNVHelpFormatter)
 
@@ -67,6 +73,8 @@
     args = parser.parse_args()
     gcnvkernel.cli_commons.set_logging_config_from_args(args)
 
+    logger.info("THEANO_FLAGS environment variable has been set to: {theano_flags}".format(theano_flags=theano_flags))
+
     # check gcnvkernel version in the input model path
     gcnvkernel.io_commons.check_gcnvkernel_version_from_path(args.input_model_path)
 
diff --git a/src/main/resources/org/broadinstitute/hellbender/tools/copynumber/cohort_denoising_calling.py b/src/main/resources/org/broadinstitute/hellbender/tools/copynumber/cohort_denoising_calling.py
index aa1ccfd2348..20927c3f55d 100644
--- a/src/main/resources/org/broadinstitute/hellbender/tools/copynumber/cohort_denoising_calling.py
+++ b/src/main/resources/org/broadinstitute/hellbender/tools/copynumber/cohort_denoising_calling.py
@@ -1,13 +1,16 @@
 import os
-import shutil
 
 # set theano flags
-os.environ["THEANO_FLAGS"] = "device=cpu,floatX=float64,optimizer=fast_run,compute_test_value=ignore," + \
-                             "openmp=true,blas.ldflags=-lmkl_rt,openmp_elemwise_minsize=10"
+user_theano_flags = os.environ.get("THEANO_FLAGS")
+default_theano_flags = "device=cpu,floatX=float64,optimizer=fast_run,compute_test_value=ignore," + \
+                       "openmp=true,blas.ldflags=-lmkl_rt,openmp_elemwise_minsize=10"
+theano_flags = default_theano_flags + ("" if user_theano_flags is None else "," + user_theano_flags)
+os.environ["THEANO_FLAGS"] = theano_flags
 
 import logging
 import argparse
 import gcnvkernel
+import shutil
 
 logger = logging.getLogger("cohort_denoising_calling")
 
@@ -97,6 +100,8 @@
     args = parser.parse_args()
     gcnvkernel.cli_commons.set_logging_config_from_args(args)
 
+    logger.info("THEANO_FLAGS environment variable has been set to: {theano_flags}".format(theano_flags=theano_flags))
+
     # load modeling interval list
     modeling_interval_list = gcnvkernel.io_intervals_and_counts.load_interval_list_tsv_file(args.modeling_interval_list)
 
diff --git a/src/main/resources/org/broadinstitute/hellbender/tools/copynumber/cohort_determine_ploidy_and_depth.py b/src/main/resources/org/broadinstitute/hellbender/tools/copynumber/cohort_determine_ploidy_and_depth.py
index 1c557aac27d..6343776c37b 100644
--- a/src/main/resources/org/broadinstitute/hellbender/tools/copynumber/cohort_determine_ploidy_and_depth.py
+++ b/src/main/resources/org/broadinstitute/hellbender/tools/copynumber/cohort_determine_ploidy_and_depth.py
@@ -1,13 +1,19 @@
 import os
 
 # set theano flags
-os.environ["THEANO_FLAGS"] = "device=cpu,floatX=float64,optimizer=fast_run,compute_test_value=ignore," + \
-                             "openmp=true,blas.ldflags=-lmkl_rt,openmp_elemwise_minsize=10"
+user_theano_flags = os.environ.get("THEANO_FLAGS")
+default_theano_flags = "device=cpu,floatX=float64,optimizer=fast_run,compute_test_value=ignore," + \
+                       "openmp=true,blas.ldflags=-lmkl_rt,openmp_elemwise_minsize=10"
+theano_flags = default_theano_flags + ("" if user_theano_flags is None else "," + user_theano_flags)
+os.environ["THEANO_FLAGS"] = theano_flags
 
+import logging
 import argparse
 import gcnvkernel
 import shutil
 
+logger = logging.getLogger("cohort_determine_ploidy_and_depth")
+
 parser = argparse.ArgumentParser(description="gCNV contig ploidy and read depth determination tool",
                                  formatter_class=gcnvkernel.cli_commons.GCNVHelpFormatter)
 
@@ -74,6 +80,8 @@
     args = parser.parse_args()
     gcnvkernel.cli_commons.set_logging_config_from_args(args)
 
+    logger.info("THEANO_FLAGS environment variable has been set to: {theano_flags}".format(theano_flags=theano_flags))
+
     # read contig ploidy prior map from file
     contig_ploidy_prior_map = gcnvkernel.io_ploidy.get_contig_ploidy_prior_map_from_tsv_file(
         args.contig_ploidy_prior_table)
diff --git a/src/main/resources/org/broadinstitute/hellbender/tools/copynumber/segment_gcnv_calls.py b/src/main/resources/org/broadinstitute/hellbender/tools/copynumber/segment_gcnv_calls.py
index 87f4557fd6e..cf7d94868c2 100644
--- a/src/main/resources/org/broadinstitute/hellbender/tools/copynumber/segment_gcnv_calls.py
+++ b/src/main/resources/org/broadinstitute/hellbender/tools/copynumber/segment_gcnv_calls.py
@@ -1,8 +1,11 @@
 import os
 
 # set theano flags
-os.environ["THEANO_FLAGS"] = "device=cpu,floatX=float64,optimizer=fast_run,compute_test_value=ignore," + \
-                             "openmp=true,blas.ldflags=-lmkl_rt,openmp_elemwise_minsize=10"
+user_theano_flags = os.environ.get("THEANO_FLAGS")
+default_theano_flags = "device=cpu,floatX=float64,optimizer=fast_run,compute_test_value=ignore," + \
+                       "openmp=true,blas.ldflags=-lmkl_rt,openmp_elemwise_minsize=10"
+theano_flags = default_theano_flags + ("" if user_theano_flags is None else "," + user_theano_flags)
+os.environ["THEANO_FLAGS"] = theano_flags
 
 import logging
 import argparse
@@ -57,6 +60,8 @@
     args = parser.parse_args()
     gcnvkernel.cli_commons.set_logging_config_from_args(args)
 
+    logger.info("THEANO_FLAGS environment variable has been set to: {theano_flags}".format(theano_flags=theano_flags))
+
     # load read depth and ploidy metadata
     logger.info("Loading ploidy calls...")
     logger.debug("Ploidy calls path: {0}".format(args.ploidy_calls_path))