-
Notifications
You must be signed in to change notification settings - Fork 598
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
WDLize GvsPrepareCallset (briefly known as CreateCohortTable) #7200
Changes from 16 commits
a0e4874
e3ad814
bd3469e
e01c16f
be53b43
a5dd483
8293d0f
9c6d19a
5fffcab
2dfefff
3d02841
4ba7b65
9c0a154
11d5c79
433fdfa
6d7e113
a705592
bb916f3
f2122e0
b475a9d
0d4bb2f
049909f
0ecb319
018fae3
a21626c
438b89e
9ffe9ae
2024451
8ea51c8
2c5d25e
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
{ | ||
"CreateCohortTable.project": "spec-ops-aou", | ||
"CreateCohortTable.dataset": "gvs_tieout_acmg_v1", | ||
|
||
"CreateCohortTable.docker": "us.gcr.io/broad-dsde-methods/broad-gatk-snapshots:varstore_cb56620f1db171d3f1c682e150e6aeb0cef64a83_mmt_ngs_cohort_extract_wdl_2020_04_12" | ||
} | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,86 @@ | ||
version 1.0 | ||
|
||
workflow CreateCohortTable { | ||
input { | ||
String project | ||
String dataset | ||
mmorgantaylor marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
String? docker | ||
} | ||
|
||
# TODO update this docker source | ||
String docker_final = select_first([docker, "us.gcr.io/broad-dsde-methods/variantstore:latest"]) | ||
mmorgantaylor marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
call CreateCohortTableTask { | ||
input: | ||
project = project, | ||
dataset = dataset, | ||
|
||
docker = docker_final | ||
} | ||
|
||
} | ||
|
||
task CreateCohortTableTask { | ||
# indicates that this task should NOT be call cached | ||
meta { | ||
volatile: true | ||
} | ||
|
||
input { | ||
String project | ||
String dataset | ||
|
||
String? query_project | ||
String? destination_project | ||
String? destination_dataset | ||
|
||
String? destination_cohort_table_name | ||
String? fq_cohort_sample_table | ||
String? fq_sample_mapping_table | ||
|
||
File? service_account_json | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. is this still defined as a File even if it's just a path to a json in GCP? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. as Kris pointed out, since it's a file, it'll localize, so we can just pass it straight into python. tested! |
||
String docker | ||
} | ||
|
||
#### set defaults #### | ||
String query_project_final = if defined(query_project) then "${query_project}" else "${project}" | ||
String destination_project_final = if defined(destination_project) then "${destination_project}" else "${project}" | ||
String destination_dataset_final = if defined(destination_dataset) then "${destination_dataset}" else "${dataset}" | ||
|
||
String destination_cohort_table_name_final = if defined(destination_cohort_table_name) then "${destination_cohort_table_name}" else "exported_cohort_all_samples" | ||
String fq_cohort_sample_table_final = if defined(fq_cohort_sample_table) then "${fq_cohort_sample_table}" else "${project}.${dataset}.sample_info" | ||
String fq_sample_mapping_table_final = if defined(fq_sample_mapping_table) then "${fq_sample_mapping_table}" else "${project}.${dataset}.sample_info" | ||
|
||
String has_service_account_file = if (defined(service_account_json)) then 'true' else 'false' | ||
|
||
command <<< | ||
set -e | ||
|
||
if [ ~{has_service_account_file} = 'true' ]; then | ||
gcloud auth activate-service-account --key-file='~{service_account_json}' | ||
fi | ||
|
||
python3 /app/create_cohort_data_table.py \ | ||
--fq_petvet_dataset ~{project}.~{dataset} \ | ||
--fq_temp_table_dataset ~{destination_project_final}.temp_tables \ | ||
--fq_destination_dataset ~{destination_project_final}.~{destination_dataset_final} \ | ||
--destination_table ~{destination_cohort_table_name_final} \ | ||
--fq_cohort_sample_names ~{fq_cohort_sample_table_final} \ | ||
--query_project ~{query_project_final} \ | ||
--fq_sample_mapping_table ~{fq_sample_mapping_table_final} | ||
>>> | ||
|
||
runtime { | ||
docker: docker | ||
memory: "10 GB" | ||
disks: "local-disk 100 HDD" | ||
bootDiskSizeGb: 15 | ||
preemptible: 0 | ||
cpu: 1 | ||
} | ||
|
||
} | ||
|
||
|
||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,14 +1,16 @@ | ||
FROM python:3.7 | ||
FROM gcr.io/google.com/cloudsdktool/cloud-sdk:305.0.0 | ||
# FROM python:3.7 | ||
|
||
# Copy the application's requirements.txt and run pip to install | ||
ADD requirements.txt /app/requirements.txt | ||
RUN pip install -r /app/requirements.txt | ||
|
||
# Add the application source code. | ||
ADD raw_array_cohort_extract.py /app | ||
mmorgantaylor marked this conversation as resolved.
Show resolved
Hide resolved
|
||
ADD ngs_cohort_extract.py /app | ||
ADD create_cohort_data_table.py /app | ||
|
||
# install google SDK | ||
RUN curl -sSL https://sdk.cloud.google.com | bash | ||
|
||
WORKDIR /app | ||
ENTRYPOINT ["/bin/bash"] |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,14 @@ | ||
if [ $# -lt 1 ]; then | ||
echo "USAGE: ./build_docker.sh [DOCKER_TAG_STRING]" | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. ❤️ |
||
echo " e.g.: ./build_docker.sh mybranch_2021_04_03" | ||
exit 1 | ||
fi | ||
|
||
INFO=$1 | ||
GCR_TAG="us.gcr.io/broad-dsde-methods/variantstore:${INFO}" | ||
|
||
docker build . -t broad-dsde-methods/variantstore:${INFO} | ||
docker tag broad-dsde-methods/variantstore:${INFO} ${GCR_TAG} | ||
docker push ${GCR_TAG} | ||
|
||
echo "docker image pushed to \"${GCR_TAG}\"" |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
remove before merging