diff --git a/scripts/funcotator/testing/compareTwoReferenceDictionaries.sh b/scripts/funcotator/testing/compareTwoReferenceDictionaries.sh
index cc36bba50ce..7d26a74b95b 100755
--- a/scripts/funcotator/testing/compareTwoReferenceDictionaries.sh
+++ b/scripts/funcotator/testing/compareTwoReferenceDictionaries.sh
@@ -32,7 +32,7 @@ MAXARGS=0
################################################################################
# Change these to point to reference dictionaries:
-D="/Users/jonn/Development/references"
+D="."
#refAFile="${D}/GRCh37.p13.genome.dict"
#refAFile="${D}/human_g1k_v37.dict"
refAFile="${D}/ucsc.hg19.dict"
diff --git a/src/main/java/org/broadinstitute/hellbender/tools/funcotator/Funcotator.java b/src/main/java/org/broadinstitute/hellbender/tools/funcotator/Funcotator.java
index adbc5f7a740..847978a783a 100644
--- a/src/main/java/org/broadinstitute/hellbender/tools/funcotator/Funcotator.java
+++ b/src/main/java/org/broadinstitute/hellbender/tools/funcotator/Funcotator.java
@@ -745,11 +745,14 @@ public boolean requiresReference() {
@Override
public void onTraversalStart() {
- logger.info("Validating Sequence Dictionaries...");
if (seqValidationArguments.performSequenceDictionaryValidation()) {
+ logger.info("Validating sequence dictionaries...");
// Ensure that the reference dictionary is a superset of the variant dictionary:
checkReferenceDictionaryIsSupersetOfVariantDictionary();
}
+ else {
+ logger.info("Skipping sequence dictionary validation.");
+ }
logger.info("Processing user transcripts/defaults/overrides...");
Utils.validateArg(funcotatorArgs.outputFormatType != FuncotatorArgumentDefinitions.OutputFormatType.SEG,
diff --git a/src/main/java/org/broadinstitute/hellbender/tools/funcotator/FuncotatorDataSourceDownloader.java b/src/main/java/org/broadinstitute/hellbender/tools/funcotator/FuncotatorDataSourceDownloader.java
index 01c071e1d2a..8468594d506 100644
--- a/src/main/java/org/broadinstitute/hellbender/tools/funcotator/FuncotatorDataSourceDownloader.java
+++ b/src/main/java/org/broadinstitute/hellbender/tools/funcotator/FuncotatorDataSourceDownloader.java
@@ -9,6 +9,7 @@
import org.broadinstitute.hellbender.cmdline.CommandLineProgram;
import org.broadinstitute.hellbender.cmdline.StandardArgumentDefinitions;
import org.broadinstitute.hellbender.exceptions.UserException;
+import org.broadinstitute.hellbender.tools.funcotator.dataSources.DataSourceUtils;
import org.broadinstitute.hellbender.utils.io.IOUtils;
import org.broadinstitute.hellbender.utils.nio.NioFileCopierWithProgressMeter;
import org.broadinstitute.hellbender.utils.nio.NioFileCopierWithProgressMeterResults;
@@ -25,7 +26,7 @@
*
General Information
*
* This tool can download pre-packaged data sources for both the somatic and germline use cases.
- * The data sources downloaded by this tool correspond to the current minimum of the data sources supported as defined in {@link org.broadinstitute.hellbender.tools.funcotator.dataSources.DataSourceUtils#CURRENT_MINIMUM_DATA_SOURCE_VERSION}.
+ * The data sources downloaded by this tool correspond to the latest / current maximum of the data sources supported as defined in {@link org.broadinstitute.hellbender.tools.funcotator.dataSources.DataSourceUtils#CURRENT_MAXIMUM_DATA_SOURCE_VERSION}.
*
*
*
@@ -71,17 +72,19 @@ public class FuncotatorDataSourceDownloader extends CommandLineProgram {
//==================================================================================================================
// Private Static Members:
- private static final String BASE_URL = "gs://broad-public-datasets/funcotator/funcotator_dataSources.v1.7.20200521";
+ // Set to always get the latest version of the data sources:
+ private static final String BASE_URL = DataSourceUtils.DATA_SOURCES_BUCKET_PATH +
+ DataSourceUtils.DATA_SOURCES_NAME_PREFIX + "." + DataSourceUtils.getDataSourceMaxVersionString();
- private static final String GERMLINE_GCLOUD_DATASOURCES_BASEURL = BASE_URL + "g";
+ private static final String GERMLINE_GCLOUD_DATASOURCES_BASEURL = BASE_URL + DataSourceUtils.DS_GERMLINE_NAME_MODIFIER;
@VisibleForTesting
- static final Path GERMLINE_GCLOUD_DATASOURCES_PATH = IOUtils.getPath(GERMLINE_GCLOUD_DATASOURCES_BASEURL + ".tar.gz");
- private static final Path GERMLINE_GCLOUD_DATASOURCES_SHA256_PATH = IOUtils.getPath(GERMLINE_GCLOUD_DATASOURCES_BASEURL + ".sha256");
+ static final Path GERMLINE_GCLOUD_DATASOURCES_PATH = IOUtils.getPath(GERMLINE_GCLOUD_DATASOURCES_BASEURL + DataSourceUtils.DS_EXTENSION);
+ private static final Path GERMLINE_GCLOUD_DATASOURCES_SHA256_PATH = IOUtils.getPath(GERMLINE_GCLOUD_DATASOURCES_BASEURL + DataSourceUtils.DS_CHECKSUM_EXTENSION);
- public static final String SOMATIC_GCLOUD_DATASOURCES_BASEURL = BASE_URL + "s";
+ public static final String SOMATIC_GCLOUD_DATASOURCES_BASEURL = BASE_URL + DataSourceUtils.DS_SOMATIC_NAME_MODIFIER;;
- public static final Path SOMATIC_GCLOUD_DATASOURCES_PATH = IOUtils.getPath(SOMATIC_GCLOUD_DATASOURCES_BASEURL + ".tar.gz");
- private static final Path SOMATIC_GCLOUD_DATASOURCES_SHA256_PATH = IOUtils.getPath(SOMATIC_GCLOUD_DATASOURCES_BASEURL + ".sha256");
+ public static final Path SOMATIC_GCLOUD_DATASOURCES_PATH = IOUtils.getPath(SOMATIC_GCLOUD_DATASOURCES_BASEURL + DataSourceUtils.DS_EXTENSION);
+ private static final Path SOMATIC_GCLOUD_DATASOURCES_SHA256_PATH = IOUtils.getPath(SOMATIC_GCLOUD_DATASOURCES_BASEURL + DataSourceUtils.DS_CHECKSUM_EXTENSION);
//==================================================================================================================
// Private Members:
diff --git a/src/main/java/org/broadinstitute/hellbender/tools/funcotator/FuncotatorEngine.java b/src/main/java/org/broadinstitute/hellbender/tools/funcotator/FuncotatorEngine.java
index 545936296ac..8b036b91fa5 100644
--- a/src/main/java/org/broadinstitute/hellbender/tools/funcotator/FuncotatorEngine.java
+++ b/src/main/java/org/broadinstitute/hellbender/tools/funcotator/FuncotatorEngine.java
@@ -471,10 +471,21 @@ private boolean determineReferenceAndDatasourceCompatibility() {
boolean mustConvertInputContigsToHg19 = false;
- if ( funcotatorArgs.forceB37ToHg19ContigNameConversion ||
- ( funcotatorArgs.referenceVersion.equals(BaseFuncotatorArgumentCollection.FuncotatorReferenceVersionHg19) &&
- FuncotatorUtils.isSequenceDictionaryUsingB37Reference(sequenceDictionaryForDrivingVariants) )) {
+ // Do individual checks here so we can have a helpful log message for each case:
+ if ( funcotatorArgs.forceB37ToHg19ContigNameConversion ) {
+ logger.info("Forcing B37 -> HG19 Variant conversion.");
+ mustConvertInputContigsToHg19 = true;
+ }
+ else if ( funcotatorArgs.referenceVersion.equals(BaseFuncotatorArgumentCollection.FuncotatorReferenceVersionHg19) &&
+ FuncotatorUtils.isSequenceDictionaryUsingB37Reference(sequenceDictionaryForDrivingVariants) ) {
+ logger.info("VCF sequence dictionary detected as B37 in HG19 annotation mode. Performing conversion.");
+ mustConvertInputContigsToHg19 = true;
+ }
+ else {
+ logger.info("Using given VCF and Reference. No conversion required.");
+ }
+ if (mustConvertInputContigsToHg19) {
// NOTE AND WARNING:
// hg19 is from ucsc. b37 is from the genome reference consortium.
// ucsc decided the grc version had some bad data in it, so they blocked out some of the bases, aka "masked" them
@@ -487,8 +498,6 @@ private boolean determineReferenceAndDatasourceCompatibility() {
logger.warn("WARNING: You are using B37 as a reference. " +
"Funcotator will convert your variants to GRCh37, and this will be fine in the vast majority of cases. " +
"There MAY be some errors (e.g. in the Y chromosome, but possibly in other places as well) due to changes between the two references.");
-
- mustConvertInputContigsToHg19 = true;
}
return mustConvertInputContigsToHg19;
diff --git a/src/main/java/org/broadinstitute/hellbender/tools/funcotator/dataSources/DataSourceUtils.java b/src/main/java/org/broadinstitute/hellbender/tools/funcotator/dataSources/DataSourceUtils.java
index 8937ced993f..4bac84acc05 100644
--- a/src/main/java/org/broadinstitute/hellbender/tools/funcotator/dataSources/DataSourceUtils.java
+++ b/src/main/java/org/broadinstitute/hellbender/tools/funcotator/dataSources/DataSourceUtils.java
@@ -23,6 +23,8 @@
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.*;
+import java.time.LocalDate;
+import java.time.Month;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -57,24 +59,37 @@ private DataSourceUtils() {}
// Track our minimum version number here:
@VisibleForTesting
- static final int MIN_MAJOR_VERSION_NUMBER = 1;
+ static final int MIN_MAJOR_VERSION_NUMBER = 1;
@VisibleForTesting
- static final int MIN_MINOR_VERSION_NUMBER = 6;
+ static final int MIN_MINOR_VERSION_NUMBER = 6;
@VisibleForTesting
- static final int MIN_YEAR_RELEASED = 2019;
+ static final LocalDate MIN_DATE = LocalDate.of(2019, Month.JANUARY, 24);
+
+ // Track out maximum version number here:
+ @VisibleForTesting
+ static final int MAX_MAJOR_VERSION_NUMBER = 1;
@VisibleForTesting
- static final int MIN_MONTH_RELEASED = 1;
+ static final int MAX_MINOR_VERSION_NUMBER = 7;
@VisibleForTesting
- static final int MIN_DAY_RELEASED = 24;
+ static final LocalDate MAX_DATE = LocalDate.of(2020, Month.MAY, 21);
//==================================================================================================================
// Public Static Members:
/** The minimum version of the data sources required for funcotator to run. */
- public static final String CURRENT_MINIMUM_DATA_SOURCE_VERSION = String.format("v%d.%d.%d%02d%02d", MIN_MAJOR_VERSION_NUMBER, MIN_MINOR_VERSION_NUMBER, MIN_YEAR_RELEASED, MIN_MONTH_RELEASED, MIN_DAY_RELEASED);
+ public static final String CURRENT_MINIMUM_DATA_SOURCE_VERSION = getDataSourceMinVersionString();
+
+ /** The maximum supported version of the data sources for funcotator to run. */
+ public static final String CURRENT_MAXIMUM_DATA_SOURCE_VERSION = getDataSourceMaxVersionString();
+
public static final String MANIFEST_FILE_NAME = "MANIFEST.txt";
public static final String DATA_SOURCES_FTP_PATH = "ftp://gsapubftp-anonymous@ftp.broadinstitute.org/bundle/funcotator/";
public static final String DATA_SOURCES_BUCKET_PATH = "gs://broad-public-datasets/funcotator/";
+ public static final String DATA_SOURCES_NAME_PREFIX = "funcotator_dataSources";
+ public static final String DS_SOMATIC_NAME_MODIFIER = "s";
+ public static final String DS_GERMLINE_NAME_MODIFIER = "g";
+ public static final String DS_EXTENSION = ".tar.gz";
+ public static final String DS_CHECKSUM_EXTENSION = ".sha256";
// TODO: Turn these into an enum (Issue #5465 - https://github.com/broadinstitute/gatk/issues/5465):
public static final String CONFIG_FILE_FIELD_NAME_NAME = "name";
@@ -100,6 +115,51 @@ private DataSourceUtils() {}
//==================================================================================================================
// Public Static Methods:
+ /**
+ * Get the string representing the Min version information for funcotator as it would be written in the data sources
+ * release files.
+ * Max version info is specified in the following variables:
+ * {@link #MIN_MAJOR_VERSION_NUMBER}
+ * {@link #MIN_MINOR_VERSION_NUMBER}
+ * {@link #MIN_DATE}
+ * @return A {@link String} representing the Min version information as it would appear in the data sources file name.
+ */
+ public static String getDataSourceMinVersionString() {
+ return getDataSourceVersionString(MIN_MAJOR_VERSION_NUMBER, MIN_MINOR_VERSION_NUMBER, MIN_DATE);
+ }
+
+ /**
+ * Get the string representing the Max version information for funcotator as it would be written in the data sources
+ * release files.
+ * Max version info is specified in the following variables:
+ * {@link #MAX_MAJOR_VERSION_NUMBER}
+ * {@link #MAX_MINOR_VERSION_NUMBER}
+ * {@link #MAX_DATE}
+ * @return A {@link String} representing the Max version information as it would appear in the data sources file name.
+ */
+ public static String getDataSourceMaxVersionString() {
+ return getDataSourceVersionString(MAX_MAJOR_VERSION_NUMBER, MAX_MINOR_VERSION_NUMBER, MAX_DATE);
+ }
+
+
+ /**
+ * Get the string representing the given version information for funcotator as it would be written in the data sources
+ * release files.
+ * @param major {@code int} representing the major version of the data sources to use.
+ * @param minor {@code int} representing the minor version of the data sources to use.
+ * @param date {@link LocalDate} representing the date of the data sources to use.
+ * @return A {@link String} representing the given version information as it would appear in the data sources file name.
+ */
+ public static String getDataSourceVersionString(final int major, final int minor, final LocalDate date) {
+ return String.format("v%d.%d.%d%02d%02d",
+ major,
+ minor,
+ date.getYear(),
+ date.getMonthValue(),
+ date.getDayOfMonth()
+ );
+ }
+
/**
* Initializes the data sources for {@link Funcotator}.
* @param refVersion The version of the reference we're using to create annotations. Must not be {@code null}.
@@ -196,7 +256,7 @@ else if ( !hasGencodeDataSource ) {
* @param directory The {@link Path} of the directory in which to search for a config file. Must not be {@code null}.
* @return The {@link Path} to the config file found in the given {@code directory}.
*/
- public static Path getConfigfile(final Path directory) {
+ private static Path getConfigfile(final Path directory) {
Utils.nonNull(directory);
@@ -225,7 +285,7 @@ else if ( configFileSet.size() == 0 ) {
}
/** @return {@code true} if the given {@link Path} exists, is readable, and is a directory; {@code false} otherwise. */
- public static boolean isValidDirectory(final Path p) {
+ private static boolean isValidDirectory(final Path p) {
Utils.nonNull(p);
return Files.exists(p) && Files.isReadable(p) && Files.isDirectory(p);
}
@@ -628,7 +688,7 @@ private static boolean logDataSourcesInfo(final Path dataSourcesPath) {
Integer versionYear = null;
Integer versionMonth = null;
Integer versionDay = null;
- String versionDecorator = null;
+ String versionDecorator;
String source = null;
String alternateSource = null;
@@ -711,10 +771,12 @@ private static boolean logDataSourcesInfo(final Path dataSourcesPath) {
// Warn the user if they need newer stuff.
if ( !dataSourcesPathIsAcceptable ) {
-
String message = "";
- message = message + "ERROR: Given data source path is too old! Minimum required version is: " + CURRENT_MINIMUM_DATA_SOURCE_VERSION + " (yours: " + version + ")\n";
- message = message + " You must download a newer version of the data sources from the Broad Institute FTP site: " + DATA_SOURCES_FTP_PATH + "\n";
+ message = message + "ERROR: Given data source path is too old or too new! \n";
+ message = message + " Minimum required version is: " + CURRENT_MINIMUM_DATA_SOURCE_VERSION + "\n";
+ message = message + " Maximum allowed version is: " + CURRENT_MAXIMUM_DATA_SOURCE_VERSION + "\n";
+ message = message + " Yours: " + version + "\n";
+ message = message + " You must download a compatible version of the data sources from the Broad Institute FTP site: " + DATA_SOURCES_FTP_PATH + "\n";
message = message + " or the Broad Institute Google Bucket: " + DATA_SOURCES_BUCKET_PATH + "\n";
throw new UserException( message );
}
@@ -722,35 +784,38 @@ private static boolean logDataSourcesInfo(final Path dataSourcesPath) {
return dataSourcesPathIsAcceptable;
}
+ /**
+ * Checks that the version information given is within the valid range for data source versions.
+ *
+ * @param major int containing the major version number to be checked.
+ * @param minor int containing the minor version number to be checked.
+ * @param year int containing the year version number to be checked.RecQ DNA helicase WRN
+ * @param month int containing the month version number to be checked.
+ * @param day int containing the day version number to be checked.
+ *
+ * @return {@code true} iff the given version information is valid for the current data source ranges. {@code false} otherwise.
+ */
@VisibleForTesting
static boolean validateVersionInformation(final int major, final int minor, final int year, final int month, final int day) {
- // Compare from largest to smallest differences:
-
- if ( major < MIN_MAJOR_VERSION_NUMBER ) {
+ // Compare Major Version:
+ if ((major < MIN_MAJOR_VERSION_NUMBER) || (major > MAX_MAJOR_VERSION_NUMBER)) {
return false;
}
- if ( minor < MIN_MINOR_VERSION_NUMBER ) {
+ // Compare minor version if we're on the edge of versions:
+ if ( major == MIN_MAJOR_VERSION_NUMBER && minor < MIN_MINOR_VERSION_NUMBER ) {
return false;
}
-
- if ( year < MIN_YEAR_RELEASED ) {
+ if ( major == MAX_MAJOR_VERSION_NUMBER && minor > MAX_MINOR_VERSION_NUMBER ) {
return false;
}
- else if ( year == MIN_YEAR_RELEASED ) {
- if ( month < MIN_MONTH_RELEASED ) {
- return false;
- }
- else if ( month == MIN_MONTH_RELEASED ) {
- if ( day < MIN_DAY_RELEASED ) {
- return false;
- }
- }
- }
+ // Now make sure the date is between or equal to the min and max date:
+ final LocalDate versionDate = LocalDate.of(year, month, day);
- return true;
+ // A valid date is between min and max date inclusive.
+ return (!versionDate.isBefore(MIN_DATE)) && (!versionDate.isAfter(MAX_DATE));
}
// ========================================================================================================
diff --git a/src/main/java/org/broadinstitute/hellbender/utils/SequenceDictionaryUtils.java b/src/main/java/org/broadinstitute/hellbender/utils/SequenceDictionaryUtils.java
index f836947a291..7f1db9fd945 100644
--- a/src/main/java/org/broadinstitute/hellbender/utils/SequenceDictionaryUtils.java
+++ b/src/main/java/org/broadinstitute/hellbender/utils/SequenceDictionaryUtils.java
@@ -154,6 +154,8 @@ public static void validateDictionaries( final String name1,
switch ( type ) {
case IDENTICAL:
return;
+ case SUPERSET:
+ return;
case COMMON_SUBSET:
if ( requireSuperset ) {
final Set contigs1 = dict1.getSequences().stream().map(SAMSequenceRecord::getSequenceName).collect(Collectors.toSet());
@@ -164,27 +166,25 @@ public static void validateDictionaries( final String name1,
throw new UserException.IncompatibleSequenceDictionaries(String.format("Dictionary %s is missing contigs found in dictionary %s. Missing contigs: \n %s \n", name1, name2, String.join(", ", missingContigs)), name1, dict1, name2, dict2);
}
return;
- case SUPERSET:
- return;
case NO_COMMON_CONTIGS:
throw new UserException.IncompatibleSequenceDictionaries("No overlapping contigs found", name1, dict1, name2, dict2);
case UNEQUAL_COMMON_CONTIGS: {
- List x = findDisequalCommonContigs(getCommonContigsByName(dict1, dict2), dict1, dict2);
- SAMSequenceRecord elt1 = x.get(0);
- SAMSequenceRecord elt2 = x.get(1);
- UserException ex = new UserException.IncompatibleSequenceDictionaries(String.format("Found contigs with the same name but different lengths:\n contig %s = %s / %d\n contig %s = %s / %d",
+ final List x = findDisequalCommonContigs(getCommonContigsByName(dict1, dict2), dict1, dict2);
+ final SAMSequenceRecord elt1 = x.get(0);
+ final SAMSequenceRecord elt2 = x.get(1);
+ throw new UserException.IncompatibleSequenceDictionaries(
+ String.format("Found contigs with the same name but different lengths:\n contig %s = %s / %d\n contig %s = %s / %d",
name1, elt1.getSequenceName(), elt1.getSequenceLength(),
name2, elt2.getSequenceName(), elt2.getSequenceLength()),
- name1, dict1, name2, dict2);
-
- throw ex;
+ name1, dict1, name2, dict2
+ );
}
case NON_CANONICAL_HUMAN_ORDER: {
// We only get NON_CANONICAL_HUMAN_ORDER if the caller explicitly requested that we check contig ordering,
// so we should always throw when we see it.
- UserException ex;
+ final UserException ex;
if ( nonCanonicalHumanContigOrder(dict1) ) {
ex = new UserException.LexicographicallySortedSequenceDictionary(name1, dict1);
}
@@ -198,13 +198,12 @@ public static void validateDictionaries( final String name1,
case OUT_OF_ORDER: {
// We only get OUT_OF_ORDER if the caller explicitly requested that we check contig ordering,
// so we should always throw when we see it.
- UserException ex = new UserException.IncompatibleSequenceDictionaries(
+ throw new UserException.IncompatibleSequenceDictionaries(
"The relative ordering of the common contigs in " + name1 + " and " + name2 +
" is not the same; to fix this please see: "
+ "(https://www.broadinstitute.org/gatk/guide/article?id=1328), "
+ " which describes reordering contigs in BAM and VCF files.",
name1, dict1, name2, dict2);
- throw ex;
}
case DIFFERENT_INDICES: {
@@ -215,8 +214,7 @@ public static void validateDictionaries( final String name1,
"that is sensitive to contig ordering can fail when this is the case. " +
"You should fix the sequence dictionaries so that all shared contigs " +
"occur at the same absolute positions in both dictionaries.";
- final UserException ex = new UserException.IncompatibleSequenceDictionaries(msg, name1, dict1, name2, dict2);
- throw ex;
+ throw new UserException.IncompatibleSequenceDictionaries(msg, name1, dict1, name2, dict2);
}
default:
throw new GATKException("Unexpected SequenceDictionaryComparison type: " + type);
diff --git a/src/main/java/org/broadinstitute/hellbender/utils/Utils.java b/src/main/java/org/broadinstitute/hellbender/utils/Utils.java
index 271c89d659d..a78325b8c4a 100644
--- a/src/main/java/org/broadinstitute/hellbender/utils/Utils.java
+++ b/src/main/java/org/broadinstitute/hellbender/utils/Utils.java
@@ -4,16 +4,13 @@
import com.google.common.collect.Iterators;
import com.google.common.primitives.Ints;
import htsjdk.samtools.SAMFileHeader;
-import htsjdk.samtools.util.Locatable;
import htsjdk.tribble.util.ParsingUtils;
-import htsjdk.variant.variantcontext.VariantContext;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.math3.random.RandomDataGenerator;
import org.apache.commons.math3.random.Well19937c;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
-import org.broadinstitute.hellbender.engine.FeatureDataSource;
import org.broadinstitute.hellbender.exceptions.GATKException;
import org.broadinstitute.hellbender.utils.param.ParamUtils;
diff --git a/src/test/java/org/broadinstitute/hellbender/tools/funcotator/dataSources/DataSourceUtilsUnitTest.java b/src/test/java/org/broadinstitute/hellbender/tools/funcotator/dataSources/DataSourceUtilsUnitTest.java
index 2b9725a01fb..442281e3a4d 100644
--- a/src/test/java/org/broadinstitute/hellbender/tools/funcotator/dataSources/DataSourceUtilsUnitTest.java
+++ b/src/test/java/org/broadinstitute/hellbender/tools/funcotator/dataSources/DataSourceUtilsUnitTest.java
@@ -2,11 +2,16 @@
import org.apache.commons.lang.RandomStringUtils;
import org.broadinstitute.hellbender.GATKBaseTest;
+import org.broadinstitute.hellbender.utils.io.IOUtils;
import org.testng.Assert;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
+import java.time.LocalDate;
+import java.time.Month;
+import java.time.YearMonth;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Matcher;
@@ -26,44 +31,280 @@ public class DataSourceUtilsUnitTest extends GATKBaseTest {
//==================================================================================================================
// Helper Methods:
+ /**
+ * Add a test case to the given {@code testArgs} for version-based tests.
+ *
+ * Calculates the expected value based on the validity of the input arguments and the valid data ranges in
+ * {@link DataSourceUtils}.
+ *
+ * @param testArgs {@code List} test arguments list to which to add the test case.
+ * @param maj {@code int} representing the major version to test.
+ * @param min {@code int} representing the minor version to test.
+ * @param date {@link LocalDate} represeting the date of the version to test.
+ */
+ private void addVersionTestCase(final List testArgs,
+ final int maj,
+ final int min,
+ final LocalDate date) {
+ testArgs.add(
+ new Object[]{
+ maj, min, date,
+ (
+ (maj >= DataSourceUtils.MIN_MAJOR_VERSION_NUMBER) &&
+ (maj <= DataSourceUtils.MAX_MAJOR_VERSION_NUMBER) &&
+ (maj != DataSourceUtils.MIN_MAJOR_VERSION_NUMBER || min >= DataSourceUtils.MIN_MINOR_VERSION_NUMBER) &&
+ (maj != DataSourceUtils.MAX_MAJOR_VERSION_NUMBER || min <= DataSourceUtils.MAX_MINOR_VERSION_NUMBER)
+ )
+ &&
+ (
+ (date.isAfter(DataSourceUtils.MIN_DATE) || date.isEqual(DataSourceUtils.MIN_DATE)) &&
+ (date.isBefore(DataSourceUtils.MAX_DATE) || date.isEqual(DataSourceUtils.MAX_DATE))
+ )
+ }
+ );
+ }
+
+ /**
+ * Add a test case to the given {@code testArgs} for version-based tests.
+ * @param testArgs {@code List} test arguments list to which to add the test case.
+ * @param maj {@code int} representing the major version to test.
+ * @param min {@code int} representing the minor version to test.
+ * @param date {@link LocalDate} represeting the date of the version to test.
+ * @param expected The expected {@code boolean} value for the test case given the input version data.
+ */
+ private void addVersionTestCase(final List testArgs,
+ final int maj,
+ final int min,
+ final LocalDate date,
+ final boolean expected) {
+ testArgs.add(new Object[]{maj, min, date, expected});
+ }
+
private List createBaseTestVersionData() {
final ArrayList testArgs = new ArrayList<>();
- final ArrayList baseArgs = new ArrayList<>();
- baseArgs.add(DataSourceUtils.MIN_MAJOR_VERSION_NUMBER);
- baseArgs.add(DataSourceUtils.MIN_MINOR_VERSION_NUMBER);
- baseArgs.add(DataSourceUtils.MIN_YEAR_RELEASED);
- baseArgs.add(DataSourceUtils.MIN_MONTH_RELEASED);
- baseArgs.add(DataSourceUtils.MIN_DAY_RELEASED);
+ // =============================================================
+ // First do tests that check the major / minor version numbers:
+ // -------------------------------------------------------------
+
+ // major < MIN_MAJOR (minor shouldn't matter)
+ addVersionTestCase(testArgs, DataSourceUtils.MIN_MAJOR_VERSION_NUMBER - 1, 0, DataSourceUtils.MIN_DATE, false);
+ addVersionTestCase(testArgs, DataSourceUtils.MIN_MAJOR_VERSION_NUMBER - 1, DataSourceUtils.MIN_MINOR_VERSION_NUMBER - 1, DataSourceUtils.MIN_DATE, false);
+ addVersionTestCase(testArgs, DataSourceUtils.MIN_MAJOR_VERSION_NUMBER - 1, DataSourceUtils.MIN_MINOR_VERSION_NUMBER, DataSourceUtils.MIN_DATE, false);
+ addVersionTestCase(testArgs, DataSourceUtils.MIN_MAJOR_VERSION_NUMBER - 1, DataSourceUtils.MIN_MINOR_VERSION_NUMBER + 1, DataSourceUtils.MIN_DATE, false);
+ addVersionTestCase(testArgs, DataSourceUtils.MIN_MAJOR_VERSION_NUMBER - 1, DataSourceUtils.MAX_MINOR_VERSION_NUMBER - 1, DataSourceUtils.MIN_DATE, false);
+ addVersionTestCase(testArgs, DataSourceUtils.MIN_MAJOR_VERSION_NUMBER - 1, DataSourceUtils.MAX_MINOR_VERSION_NUMBER, DataSourceUtils.MIN_DATE, false);
+ addVersionTestCase(testArgs, DataSourceUtils.MIN_MAJOR_VERSION_NUMBER - 1, DataSourceUtils.MAX_MINOR_VERSION_NUMBER + 1, DataSourceUtils.MIN_DATE, false);
+
+ // major == MIN_MAJOR (minor does matter)
+ // Truth computed in `addVersionTestCase` method:
+ addVersionTestCase(testArgs, DataSourceUtils.MIN_MAJOR_VERSION_NUMBER, 0, DataSourceUtils.MIN_DATE);
+ addVersionTestCase(testArgs, DataSourceUtils.MIN_MAJOR_VERSION_NUMBER, DataSourceUtils.MIN_MINOR_VERSION_NUMBER - 1, DataSourceUtils.MIN_DATE);
+ addVersionTestCase(testArgs, DataSourceUtils.MIN_MAJOR_VERSION_NUMBER, DataSourceUtils.MIN_MINOR_VERSION_NUMBER, DataSourceUtils.MIN_DATE);
+ addVersionTestCase(testArgs, DataSourceUtils.MIN_MAJOR_VERSION_NUMBER, DataSourceUtils.MIN_MINOR_VERSION_NUMBER + 1, DataSourceUtils.MIN_DATE);
+ addVersionTestCase(testArgs, DataSourceUtils.MIN_MAJOR_VERSION_NUMBER, DataSourceUtils.MAX_MINOR_VERSION_NUMBER - 1, DataSourceUtils.MIN_DATE);
+ addVersionTestCase(testArgs, DataSourceUtils.MIN_MAJOR_VERSION_NUMBER, DataSourceUtils.MAX_MINOR_VERSION_NUMBER, DataSourceUtils.MIN_DATE);
+ addVersionTestCase(testArgs, DataSourceUtils.MIN_MAJOR_VERSION_NUMBER, DataSourceUtils.MAX_MINOR_VERSION_NUMBER + 1, DataSourceUtils.MIN_DATE);
+
+ // major > MIN_MAJOR and major < MAX_MAJOR (minor shouldn't matter)
+ if ( DataSourceUtils.MAX_MAJOR_VERSION_NUMBER - DataSourceUtils.MIN_MAJOR_VERSION_NUMBER > 1 ) {
+ addVersionTestCase(testArgs, DataSourceUtils.MIN_MAJOR_VERSION_NUMBER + 1, 0, DataSourceUtils.MIN_DATE, true);
+ addVersionTestCase(testArgs, DataSourceUtils.MIN_MAJOR_VERSION_NUMBER + 1, DataSourceUtils.MIN_MINOR_VERSION_NUMBER - 1, DataSourceUtils.MIN_DATE, true);
+ addVersionTestCase(testArgs, DataSourceUtils.MIN_MAJOR_VERSION_NUMBER + 1, DataSourceUtils.MIN_MINOR_VERSION_NUMBER, DataSourceUtils.MIN_DATE, true);
+ addVersionTestCase(testArgs, DataSourceUtils.MIN_MAJOR_VERSION_NUMBER + 1, DataSourceUtils.MIN_MINOR_VERSION_NUMBER + 1, DataSourceUtils.MIN_DATE, true);
+ addVersionTestCase(testArgs, DataSourceUtils.MIN_MAJOR_VERSION_NUMBER + 1, DataSourceUtils.MAX_MINOR_VERSION_NUMBER - 1, DataSourceUtils.MIN_DATE, true);
+ addVersionTestCase(testArgs, DataSourceUtils.MIN_MAJOR_VERSION_NUMBER + 1, DataSourceUtils.MAX_MINOR_VERSION_NUMBER, DataSourceUtils.MIN_DATE, true);
+ addVersionTestCase(testArgs, DataSourceUtils.MIN_MAJOR_VERSION_NUMBER + 1, DataSourceUtils.MAX_MINOR_VERSION_NUMBER + 1, DataSourceUtils.MIN_DATE, true);
+ }
- for ( int offset = -1 ; offset < 2; ++offset ) {
- for ( int i = 0; i < baseArgs.size(); ++i ) {
+ // major == MAX_MAJOR (minor does matter)
+ // Truth computed in `addVersionTestCase` method:
+ addVersionTestCase(testArgs, DataSourceUtils.MAX_MAJOR_VERSION_NUMBER, 0, DataSourceUtils.MIN_DATE);
+ addVersionTestCase(testArgs, DataSourceUtils.MAX_MAJOR_VERSION_NUMBER, DataSourceUtils.MIN_MINOR_VERSION_NUMBER - 1, DataSourceUtils.MIN_DATE);
+ addVersionTestCase(testArgs, DataSourceUtils.MAX_MAJOR_VERSION_NUMBER, DataSourceUtils.MIN_MINOR_VERSION_NUMBER, DataSourceUtils.MIN_DATE);
+ addVersionTestCase(testArgs, DataSourceUtils.MAX_MAJOR_VERSION_NUMBER, DataSourceUtils.MIN_MINOR_VERSION_NUMBER + 1, DataSourceUtils.MIN_DATE);
+ addVersionTestCase(testArgs, DataSourceUtils.MAX_MAJOR_VERSION_NUMBER, DataSourceUtils.MAX_MINOR_VERSION_NUMBER - 1, DataSourceUtils.MIN_DATE);
+ addVersionTestCase(testArgs, DataSourceUtils.MAX_MAJOR_VERSION_NUMBER, DataSourceUtils.MAX_MINOR_VERSION_NUMBER, DataSourceUtils.MIN_DATE);
+ addVersionTestCase(testArgs, DataSourceUtils.MAX_MAJOR_VERSION_NUMBER, DataSourceUtils.MAX_MINOR_VERSION_NUMBER + 1, DataSourceUtils.MIN_DATE);
+
+ // major > MAX_MAJOR (minor shouldn't matter)
+ addVersionTestCase(testArgs, DataSourceUtils.MAX_MAJOR_VERSION_NUMBER + 1, 0, DataSourceUtils.MAX_DATE, false);
+ addVersionTestCase(testArgs, DataSourceUtils.MAX_MAJOR_VERSION_NUMBER + 1, DataSourceUtils.MIN_MINOR_VERSION_NUMBER - 1, DataSourceUtils.MAX_DATE, false);
+ addVersionTestCase(testArgs, DataSourceUtils.MAX_MAJOR_VERSION_NUMBER + 1, DataSourceUtils.MIN_MINOR_VERSION_NUMBER, DataSourceUtils.MAX_DATE, false);
+ addVersionTestCase(testArgs, DataSourceUtils.MAX_MAJOR_VERSION_NUMBER + 1, DataSourceUtils.MIN_MINOR_VERSION_NUMBER + 1, DataSourceUtils.MAX_DATE, false);
+ addVersionTestCase(testArgs, DataSourceUtils.MAX_MAJOR_VERSION_NUMBER + 1, DataSourceUtils.MAX_MINOR_VERSION_NUMBER - 1, DataSourceUtils.MAX_DATE, false);
+ addVersionTestCase(testArgs, DataSourceUtils.MAX_MAJOR_VERSION_NUMBER + 1, DataSourceUtils.MAX_MINOR_VERSION_NUMBER, DataSourceUtils.MAX_DATE, false);
+ addVersionTestCase(testArgs, DataSourceUtils.MAX_MAJOR_VERSION_NUMBER + 1, DataSourceUtils.MAX_MINOR_VERSION_NUMBER + 1, DataSourceUtils.MAX_DATE, false);
+
+ // =============================================================
+ // Next do tests that check the dates themselves:
+ // -------------------------------------------------------------
+
+ // ----------------------------------------
+ // Dates in range:
+ // ----------------------------------------
+
+ // Min version with minimally / maximally acceptable dates:
+ testArgs.add(
+ new Object[]{
+ DataSourceUtils.MIN_MAJOR_VERSION_NUMBER,
+ DataSourceUtils.MIN_MINOR_VERSION_NUMBER,
+ DataSourceUtils.MIN_DATE,
+ true
+ }
+ );
+ testArgs.add(
+ new Object[]{
+ DataSourceUtils.MIN_MAJOR_VERSION_NUMBER,
+ DataSourceUtils.MIN_MINOR_VERSION_NUMBER,
+ DataSourceUtils.MAX_DATE,
+ true
+ }
+ );
+
+ // Max version with minimally / maximally acceptable dates:
+ testArgs.add(
+ new Object[]{
+ DataSourceUtils.MAX_MAJOR_VERSION_NUMBER,
+ DataSourceUtils.MAX_MINOR_VERSION_NUMBER,
+ DataSourceUtils.MIN_DATE,
+ true
+ }
+ );
+ testArgs.add(
+ new Object[]{
+ DataSourceUtils.MAX_MAJOR_VERSION_NUMBER,
+ DataSourceUtils.MAX_MINOR_VERSION_NUMBER,
+ DataSourceUtils.MAX_DATE,
+ true
+ }
+ );
- final ArrayList