VS 1171 Add an optional Hail whl to the integration test

.dockstore.yml

@@ -156,6 +156,7 @@ workflows:
        branches:
          - master
          - ah_var_store
+         - rc-vs-1171-add-whl
        tags:
          - /.*/
    - name: GvsExtractCallset
@@ -300,7 +301,6 @@ workflows:
          branches:
              - master
              - ah_var_store
-             - rc-vs-1102-add-vds-wdl-to-integration-tests
          tags:
              - /.*/
    - name: GvsQuickstartIntegration
@@ -310,7 +310,7 @@ workflows:
          branches:
              - master
              - ah_var_store
-             - vs_1144_phasing_revert
+             - rc-vs-1171-add-whl
          tags:
              - /.*/
    - name: GvsIngestTieout

scripts/variantstore/wdl/GvsCreateVDS.wdl

@@ -11,6 +11,7 @@ workflow GvsCreateVDS {
         String avro_path
         Boolean use_classic_VQSR = false
         String? hail_version
+        String? hail_wheel
         String cluster_prefix = "vds-cluster"
         String gcs_subnetwork_name = "subnetwork"
         String region = "us-central1"
@@ -47,6 +48,13 @@ workflow GvsCreateVDS {
     }
 
     String effective_variants_docker = select_first([variants_docker, GetToolVersions.variants_docker])
+    if (defined(hail_version) && defined(hail_wheel)) { # If neither the whl or version is specified, we go with the utils Hail version. Both cannot be specified.
+        call Utils.TerminateWorkflow as HailVersionFail {
+            input:
+                message = "A Hail version and a Hail wheel cannot both be specified",
+                basic_docker = effective_variants_docker,
+        }
+    }
     String effective_workspace_bucket = select_first([workspace_bucket, GetToolVersions.workspace_bucket])
     String effective_google_project = select_first([gcs_project, GetToolVersions.google_project])
     String effective_hail_version = select_first([hail_version, GetToolVersions.hail_version])
@@ -59,6 +67,7 @@ workflow GvsCreateVDS {
             avro_path = avro_path,
             use_classic_VQSR = use_classic_VQSR,
             hail_version = effective_hail_version,
+            hail_wheel = hail_wheel,
             gcs_project = effective_google_project,
             region = region,
             workspace_bucket = effective_workspace_bucket,
@@ -93,6 +102,7 @@ task create_vds {
         String avro_path
         Boolean use_classic_VQSR
         String? hail_version
+        String? hail_wheel
 
         String gcs_project
         String workspace_bucket
@@ -101,6 +111,7 @@ task create_vds {
 
         String variants_docker
     }
+    String hail_wheel_to_use = "hail-0.2.102-py3-none-any.whl"
 
     command <<<
         # Prepend date, time and pwd to xtrace log entries.
@@ -110,7 +121,15 @@ task create_vds {
         account_name=$(gcloud config list account --format "value(core.account)")
 
         pip3 install --upgrade pip
-        pip3 install hail~{'==' + hail_version}
+
+        if [[ -z "~{hail_wheel}" ]]
+        then
+          pip install hail~{'==' + hail_version}
+        else
+          gsutil cp ~{hail_wheel} ~{hail_wheel_to_use}
+          pip install  ~{hail_wheel_to_use}
+        fi
+
         pip3 install --upgrade google-cloud-dataproc
 
         # Generate a UUIDish random hex string of <8 hex chars (4 bytes)>-<4 hex chars (2 bytes)>

scripts/variantstore/wdl/GvsQuickstartHailIntegration.wdl

@@ -25,6 +25,7 @@ workflow GvsQuickstartHailIntegration {
 
         File? gatk_override
         String? hail_version
+        String? hail_wheel
         String expected_output_prefix
         String? sample_id_column_name ## Note that a column WILL exist that is the <entity>_id from the table name. However, some users will want to specify an alternate column for the sample_name during ingest
         String? vcf_files_column_name
@@ -48,6 +49,15 @@ workflow GvsQuickstartHailIntegration {
     }
 
     String effective_basic_docker = select_first([basic_docker, GetToolVersions.basic_docker])
+
+    if (defined(hail_version) && defined(hail_wheel)) { # If neither the whl or version is specified, we go with the utils Hail version. Both cannot be specified.
+        call Utils.TerminateWorkflow as HailVersionFail {
+            input:
+                message = "A Hail version and a Hail wheel cannot both be specified",
+                basic_docker = effective_basic_docker,
+        }
+    }
+
     String effective_cloud_sdk_docker = select_first([cloud_sdk_docker, GetToolVersions.cloud_sdk_docker])
     String effective_cloud_sdk_slim_docker = select_first([cloud_sdk_slim_docker, GetToolVersions.cloud_sdk_slim_docker])
     String effective_variants_docker = select_first([variants_docker, GetToolVersions.variants_docker])
@@ -108,6 +118,7 @@ workflow GvsQuickstartHailIntegration {
         input:
             git_branch_or_tag = git_branch_or_tag,
             hail_version = effective_hail_version,
+            hail_wheel = hail_wheel,
             use_classic_VQSR = !use_VQSR_lite,
             avro_path = GvsExtractAvroFilesForHail.avro_prefix,
             vds_destination_path = GvsExtractAvroFilesForHail.vds_output_path,
@@ -126,6 +137,8 @@ workflow GvsQuickstartHailIntegration {
             tieout_vcf_indexes = GvsQuickstartVcfIntegration.output_vcf_indexes,
             cloud_sdk_slim_docker = effective_cloud_sdk_slim_docker,
             hail_version = effective_hail_version,
+            hail_wheel = hail_wheel,
+
     }
 
     output {
@@ -149,7 +162,8 @@ task TieOutVds {
         Array[File] tieout_vcfs
         Array[File] tieout_vcf_indexes
         String cloud_sdk_slim_docker
-        String hail_version
+        String? hail_version
+        String? hail_wheel
     }
     parameter_meta {
         tieout_vcfs: {
@@ -159,6 +173,7 @@ task TieOutVds {
             localization_optional: true
         }
     }
+    String hail_wheel_to_use = "hail-0.2.102-py3-none-any.whl"
     command <<<
         # Prepend date, time and pwd to xtrace log entries.
         PS4='\D{+%F %T} \w $ '
@@ -200,7 +215,15 @@ task TieOutVds {
 
         export PYSPARK_SUBMIT_ARGS='--driver-memory 16g --executor-memory 16g pyspark-shell'
         pip install --upgrade pip
-        pip install hail==~{hail_version}
+
+
+        if [[ -z "~{hail_wheel}" ]]
+        then
+          pip install hail~{'==' + hail_version}
+        else
+          gsutil cp ~{hail_wheel} ~{hail_wheel_to_use}
+          pip install  ~{hail_wheel_to_use}
+        fi
 
         export WORK=$PWD/work
         mkdir ${WORK}

scripts/variantstore/wdl/GvsQuickstartIntegration.wdl

@@ -27,6 +27,7 @@ workflow GvsQuickstartIntegration {
         String? variants_docker
         String? gatk_docker
         String? hail_version
+        String? hail_wheel
     }
 
     File full_wgs_interval_list = "gs://gcp-public-data--broad-references/hg38/v0/wgs_calling_regions.hg38.noCentromeres.noTelomeres.interval_list"
@@ -96,6 +97,7 @@ workflow GvsQuickstartIntegration {
                 workspace_id = GetToolVersions.workspace_id,
                 submission_id = GetToolVersions.submission_id,
                 hail_version = effective_hail_version,
+                hail_wheel = hail_wheel,
         }
         call QuickstartHailIntegration.GvsQuickstartHailIntegration as GvsQuickstartHailVQSRClassicIntegration {
             input:
@@ -122,6 +124,7 @@ workflow GvsQuickstartIntegration {
                 workspace_id = GetToolVersions.workspace_id,
                 submission_id = GetToolVersions.submission_id,
                 hail_version = effective_hail_version,
+                hail_wheel = hail_wheel,
         }
 
         if (GvsQuickstartHailVQSRLiteIntegration.used_tighter_gcp_quotas) {