AoU GVS Cohort Extract wdl

scripts/variantstore/wdl/GvsExtractCallset.wdl

@@ -4,7 +4,6 @@ workflow GvsExtractCallset {
    input {
         String data_project
         String default_dataset
-        String filter_set_name
 
         File wgs_intervals
         Int scatter_count
@@ -16,10 +15,11 @@ workflow GvsExtractCallset {
         String fq_cohort_extract_table_prefix
         String query_project = data_project
 
-        String fq_filter_set_info_table = "~{data_project}.~{default_dataset}.filter_set_info"
-        String fq_filter_set_site_table = "~{data_project}.~{default_dataset}.filter_set_sites"
-        String fq_filter_set_tranches_table = "~{data_project}.~{default_dataset}.filter_set_tranches"
         Boolean do_not_filter_override = false
+        String? filter_set_name
+        String? fq_filter_set_info_table = "~{data_project}.~{default_dataset}.filter_set_info"
+        String? fq_filter_set_site_table = "~{data_project}.~{default_dataset}.filter_set_sites"
+        String? fq_filter_set_tranches_table = "~{data_project}.~{default_dataset}.filter_set_tranches"
 
         # if these are unset, default sensitivity levels will be used
         Float? snps_truth_sensitivity_filter_level_override
@@ -31,6 +31,7 @@ workflow GvsExtractCallset {
         File? service_account_json
 
         String output_file_base_name
+        String? output_gcs_dir
         File? gatk_override
     }
 
@@ -64,9 +65,15 @@ workflow GvsExtractCallset {
                 excluded_intervals       = excluded_intervals,
                 emit_pls                 = emit_pls,
                 service_account_json     = service_account_json,
-                output_file              = "${output_file_base_name}_${i}.vcf.gz"
+                output_file              = "${output_file_base_name}_${i}.vcf.gz",
+                output_gcs_dir           = output_gcs_dir
         }
     }
+
+    output {
+      Array[File] output_vcfs = ExtractTask.output_vcf
+      Array[File] output_vcf_indexes = ExtractTask.output_vcf_index
+    }
 }
 
 ################################################################################
@@ -90,14 +97,15 @@ task ExtractTask {
         String fq_cohort_extract_table
         String read_project_id
         String output_file
-        String fq_filter_set_info_table
-        String fq_filter_set_site_table
-        String fq_filter_set_tranches_table
-        String filter_set_name
-        Float? snps_truth_sensitivity_filter_level
-        Float? indels_truth_sensitivity_filter_level
+        String? output_gcs_dir
 
         Boolean do_not_filter_override
+        String? fq_filter_set_info_table
+        String? fq_filter_set_site_table
+        String? fq_filter_set_tranches_table
+        String? filter_set_name
+        Float? snps_truth_sensitivity_filter_level
+        Float? indels_truth_sensitivity_filter_level
 
         File? excluded_intervals
         Boolean? emit_pls
@@ -149,6 +157,14 @@ task ExtractTask {
                 --project-id ~{read_project_id} \
                 ~{true='--emit-pls' false='' emit_pls} \
                 ${FILTERING_ARGS}
+
+        # Drop trailing slash if one exists
+        OUTPUT_GCS_DIR=$(echo ~{output_gcs_dir} | sed 's/\/$//')
+
+        if [ -n "${OUTPUT_GCS_DIR}" ]; then
+          gsutil cp ~{output_file} ${OUTPUT_GCS_DIR}/
+          gsutil cp ~{output_file}.tbi ${OUTPUT_GCS_DIR}/
+        fi
     >>>
 
     # ------------------------------------------------

scripts/variantstore/wdl/GvsExtractCohortFromSampleNames.wdl

@@ -0,0 +1,87 @@
+version 1.0
+
+import "GvsPrepareCallset.wdl" as GvsPrepareCallset
+import "GvsExtractCallset.wdl" as GvsExtractCallset
+
+# Workflow used by AoU to extract variants for a given cohort of sample_names
+
+workflow GvsExtractCohortFromSampleNames {
+
+  input {
+    File cohort_sample_names
+    String query_project
+    String gvs_project
+    String gvs_dataset
+    String gvs_extraction_cohorts_dataset
+    String gvs_extraction_destination_dataset
+    String gvs_extraction_temp_tables_dataset
+    String extraction_uuid
+    String? output_gcs_dir
+
+    # Extract parameters
+    File wgs_intervals
+    Int scatter_count
+
+    File reference
+    File reference_index
+    File reference_dict
+
+    String output_file_base_name
+
+    Boolean do_not_filter_override = false
+    String? filter_set_name
+    String fq_filter_set_info_table = "~{gvs_project}.~{gvs_dataset}.filter_set_info"
+    String fq_filter_set_site_table = "~{gvs_project}.~{gvs_dataset}.filter_set_sites"
+    String fq_filter_set_tranches_table = "~{gvs_project}.~{gvs_dataset}.filter_set_tranches"
+
+    # if these are unset, default sensitivity levels will be used
+    Float? snps_truth_sensitivity_filter_level_override
+    Float? indels_truth_sensitivity_filter_level_override
+
+    File? gatk_override
+  }
+
+  String fq_cohort_extract_table_prefix = "~{gvs_project}.~{gvs_dataset}.~{extraction_uuid}"
+
+  call GvsPrepareCallset.GvsPrepareCallset {
+    input:
+      destination_cohort_table_prefix = fq_cohort_extract_table_prefix,
+      sample_names_to_extract         = cohort_sample_names,
+      data_project                    = query_project,
+      default_dataset                 = gvs_dataset, # unused if fq_* args are given
+      fq_petvet_dataset               = "~{gvs_project}.~{gvs_dataset}",
+      fq_sample_mapping_table         = "~{gvs_project}.~{gvs_dataset}.sample_info",
+      fq_temp_table_dataset           = gvs_extraction_temp_tables_dataset,
+      fq_destination_dataset          = gvs_extraction_destination_dataset
+  }
+
+
+  call GvsExtractCallset.GvsExtractCallset {
+    input:
+      data_project = gvs_project, # unused if fq_filter_set_* args are given or filtering is off
+      query_project = query_project,
+      default_dataset = gvs_dataset, # unused if fq_filter_set_* args are given or filtering is off
+
+      wgs_intervals = wgs_intervals,
+      scatter_count = scatter_count,
+
+      reference = reference,
+      reference_index = reference_index,
+      reference_dict = reference_dict,
+
+      fq_cohort_extract_table_prefix = fq_cohort_extract_table_prefix,
+
+      do_not_filter_override = do_not_filter_override,
+      filter_set_name = filter_set_name,
+      fq_filter_set_info_table =  fq_filter_set_info_table,
+      fq_filter_set_site_table =  fq_filter_set_site_table,
+      fq_filter_set_tranches_table =  fq_filter_set_tranches_table,
+      snps_truth_sensitivity_filter_level_override = snps_truth_sensitivity_filter_level_override,
+      indels_truth_sensitivity_filter_level_override = indels_truth_sensitivity_filter_level_override,
+
+      output_file_base_name = output_file_base_name,
+      output_gcs_dir = output_gcs_dir,
+      gatk_override = gatk_override
+  }
+
+}

scripts/variantstore/wdl/GvsPrepareCallset.wdl

@@ -40,6 +40,8 @@ workflow GvsPrepareCallset {
             docker                          = docker_final
     }
 
+    output {}
+
 }
 
 task PrepareCallsetTask {
@@ -82,6 +84,8 @@ task PrepareCallsetTask {
             ~{"--sa_key_path " + service_account_json}
     >>>
 
+    output {}
+
     runtime {
         docker: docker
         memory: "3 GB"

scripts/variantstore/wdl/extract/create_cohort_extract_data_table.py

@@ -193,7 +193,7 @@ def create_position_table(fq_temp_table_dataset, min_variant_samples):
 def make_new_pet_union_all(fq_pet_vet_dataset, fq_temp_table_dataset, sample_ids):
   def get_pet_subselect(fq_pet_table, samples, id):
     sample_stanza = ','.join([str(s) for s in samples])
-    sql = f"    q_{id} AS (SELECT p.location, p.sample_id, p.state from {fq_pet_table} p " \
+    sql = f"    q_{id} AS (SELECT p.location, p.sample_id, p.state from `{fq_pet_table}` p " \
           f"    join `{fq_temp_table_dataset}.{VET_DISTINCT_POS_TABLE}` v on (p.location = v.location) WHERE p.sample_id IN ({sample_stanza})), "
     return sql