Move BigQuery classes [VS-971] (#8375)

Move BigQuery classes [VS-971]
broadinstitute · Jun 30, 2023 · 09989de · 09989de
1 parent 17b0493
commit 09989de
Show file tree

Hide file tree

Showing 24 changed files with 37 additions and 38 deletions.
diff --git a/.dockstore.yml b/.dockstore.yml
@@ -227,6 +227,7 @@ workflows:
        branches:
          - master
          - ah_var_store
+         - vs_971_move_bigquery_classes
    - name: GvsIngestTieout
      subclass: WDL
      primaryDescriptorPath: /scripts/variantstore/wdl/GvsIngestTieout.wdl

diff --git a/src/main/java/org/broadinstitute/hellbender/tools/gvs/common/CostObservability.java b/src/main/java/org/broadinstitute/hellbender/tools/gvs/common/CostObservability.java
@@ -13,7 +13,7 @@
 import java.io.IOException;
 import java.util.Date;
 
-import static org.broadinstitute.hellbender.utils.bigquery.BigQueryUtils.extractCausalStatusRuntimeExceptionOrThrow;
+import static org.broadinstitute.hellbender.utils.gvs.bigquery.BigQueryUtils.extractCausalStatusRuntimeExceptionOrThrow;
 
 public class CostObservability {
     static final Logger logger = LogManager.getLogger(org.broadinstitute.hellbender.tools.gvs.common.CostObservability.class);

diff --git a/src/main/java/org/broadinstitute/hellbender/tools/gvs/common/FilterSensitivityTools.java b/src/main/java/org/broadinstitute/hellbender/tools/gvs/common/FilterSensitivityTools.java
@@ -6,8 +6,8 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.broadinstitute.hellbender.exceptions.UserException;
-import org.broadinstitute.hellbender.utils.bigquery.StorageAPIAvroReader;
-import org.broadinstitute.hellbender.utils.bigquery.TableReference;
+import org.broadinstitute.hellbender.utils.gvs.bigquery.StorageAPIAvroReader;
+import org.broadinstitute.hellbender.utils.gvs.bigquery.TableReference;
 import org.broadinstitute.hellbender.utils.variant.GATKVCFConstants;
 
 import java.util.HashMap;

diff --git a/src/main/java/org/broadinstitute/hellbender/tools/gvs/common/SampleList.java b/src/main/java/org/broadinstitute/hellbender/tools/gvs/common/SampleList.java
@@ -10,9 +10,9 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.broadinstitute.hellbender.exceptions.GATKException;
-import org.broadinstitute.hellbender.utils.bigquery.BigQueryUtils;
-import org.broadinstitute.hellbender.utils.bigquery.BigQueryResultAndStatistics;
-import org.broadinstitute.hellbender.utils.bigquery.TableReference;
+import org.broadinstitute.hellbender.utils.gvs.bigquery.BigQueryUtils;
+import org.broadinstitute.hellbender.utils.gvs.bigquery.BigQueryResultAndStatistics;
+import org.broadinstitute.hellbender.utils.gvs.bigquery.TableReference;
 
 public class SampleList {
     static final Logger logger = LogManager.getLogger(SampleList.class);

diff --git a/src/main/java/org/broadinstitute/hellbender/tools/gvs/extract/ExtractCohortEngine.java b/src/main/java/org/broadinstitute/hellbender/tools/gvs/extract/ExtractCohortEngine.java
@@ -18,8 +18,8 @@
 import org.broadinstitute.hellbender.tools.walkers.ReferenceConfidenceVariantContextMerger;
 import org.broadinstitute.hellbender.tools.walkers.annotator.VariantAnnotatorEngine;
 import org.broadinstitute.hellbender.utils.SimpleInterval;
-import org.broadinstitute.hellbender.utils.bigquery.StorageAPIAvroReader;
-import org.broadinstitute.hellbender.utils.bigquery.TableReference;
+import org.broadinstitute.hellbender.utils.gvs.bigquery.StorageAPIAvroReader;
+import org.broadinstitute.hellbender.utils.gvs.bigquery.TableReference;
 import org.broadinstitute.hellbender.utils.gvs.bigquery.AvroFileReader;
 import org.broadinstitute.hellbender.utils.gvs.localsort.AvroSortingCollectionCodec;
 import org.broadinstitute.hellbender.utils.gvs.localsort.SortingCollection;

diff --git a/src/main/java/org/broadinstitute/hellbender/tools/gvs/filtering/ExtractFeatures.java b/src/main/java/org/broadinstitute/hellbender/tools/gvs/filtering/ExtractFeatures.java
@@ -11,7 +11,7 @@
 import org.broadinstitute.hellbender.exceptions.UserException;
 import org.broadinstitute.hellbender.tools.gvs.common.*;
 import org.broadinstitute.hellbender.utils.SimpleInterval;
-import org.broadinstitute.hellbender.utils.bigquery.TableReference;
+import org.broadinstitute.hellbender.utils.gvs.bigquery.TableReference;
 import org.broadinstitute.hellbender.utils.io.IOUtils;
 import org.broadinstitute.hellbender.utils.variant.GATKVCFConstants;
 import org.broadinstitute.hellbender.utils.variant.GATKVCFHeaderLines;

diff --git a/src/main/java/org/broadinstitute/hellbender/tools/gvs/filtering/ExtractFeaturesBQ.java b/src/main/java/org/broadinstitute/hellbender/tools/gvs/filtering/ExtractFeaturesBQ.java
@@ -4,7 +4,7 @@
 
 import org.apache.commons.io.FileUtils;
 import org.broadinstitute.hellbender.exceptions.GATKException;
-import org.broadinstitute.hellbender.utils.bigquery.TableReference;
+import org.broadinstitute.hellbender.utils.gvs.bigquery.TableReference;
 import org.broadinstitute.hellbender.utils.io.Resource;
 
 public class ExtractFeaturesBQ {

diff --git a/src/main/java/org/broadinstitute/hellbender/tools/gvs/filtering/ExtractFeaturesEngine.java b/src/main/java/org/broadinstitute/hellbender/tools/gvs/filtering/ExtractFeaturesEngine.java
@@ -25,7 +25,10 @@
 import org.broadinstitute.hellbender.utils.GenotypeCounts;
 import org.broadinstitute.hellbender.utils.QualityUtils;
 import org.broadinstitute.hellbender.utils.SimpleInterval;
-import org.broadinstitute.hellbender.utils.bigquery.*;
+import org.broadinstitute.hellbender.utils.gvs.bigquery.BigQueryUtils;
+import org.broadinstitute.hellbender.utils.gvs.bigquery.GATKAvroReader;
+import org.broadinstitute.hellbender.utils.gvs.bigquery.StorageAPIAvroReaderAndBigQueryStatistics;
+import org.broadinstitute.hellbender.utils.gvs.bigquery.TableReference;
 import org.broadinstitute.hellbender.utils.gvs.localsort.AvroSortingCollection;
 import org.broadinstitute.hellbender.utils.gvs.localsort.SortingCollection;
 import org.broadinstitute.hellbender.utils.variant.GATKVCFConstants;

diff --git a/src/main/java/org/broadinstitute/hellbender/tools/gvs/ingest/CreateVariantIngestFiles.java b/src/main/java/org/broadinstitute/hellbender/tools/gvs/ingest/CreateVariantIngestFiles.java
@@ -22,7 +22,7 @@
 import org.broadinstitute.hellbender.utils.GenomeLocParser;
 import org.broadinstitute.hellbender.utils.GenomeLocSortedSet;
 import org.broadinstitute.hellbender.utils.IntervalUtils;
-import org.broadinstitute.hellbender.utils.bigquery.BigQueryUtils;
+import org.broadinstitute.hellbender.utils.gvs.bigquery.BigQueryUtils;
 
 import java.io.File;
 import java.io.IOException;

diff --git a/src/main/java/org/broadinstitute/hellbender/tools/gvs/ingest/LoadStatus.java b/src/main/java/org/broadinstitute/hellbender/tools/gvs/ingest/LoadStatus.java
@@ -9,14 +9,14 @@
 import org.apache.logging.log4j.Logger;
 import org.broadinstitute.hellbender.exceptions.GATKException;
 import org.broadinstitute.hellbender.tools.gvs.common.SchemaUtils;
-import org.broadinstitute.hellbender.utils.bigquery.BigQueryUtils;
-import org.broadinstitute.hellbender.utils.bigquery.BigQueryResultAndStatistics;
+import org.broadinstitute.hellbender.utils.gvs.bigquery.BigQueryUtils;
+import org.broadinstitute.hellbender.utils.gvs.bigquery.BigQueryResultAndStatistics;
 import org.json.JSONArray;
 import org.json.JSONObject;
 
 import java.io.IOException;
 
-import static org.broadinstitute.hellbender.utils.bigquery.BigQueryUtils.extractCausalStatusRuntimeExceptionOrThrow;
+import static org.broadinstitute.hellbender.utils.gvs.bigquery.BigQueryUtils.extractCausalStatusRuntimeExceptionOrThrow;
 
 public class LoadStatus {
     static final Logger logger = LogManager.getLogger(org.broadinstitute.hellbender.tools.gvs.ingest.LoadStatus.class);

diff --git a/src/main/java/org/broadinstitute/hellbender/tools/gvs/ingest/RefCreator.java b/src/main/java/org/broadinstitute/hellbender/tools/gvs/ingest/RefCreator.java
@@ -13,7 +13,7 @@
 import org.broadinstitute.hellbender.utils.GenomeLocParser;
 import org.broadinstitute.hellbender.utils.GenomeLocSortedSet;
 import org.broadinstitute.hellbender.utils.SimpleInterval;
-import org.broadinstitute.hellbender.utils.bigquery.BigQueryUtils;
+import org.broadinstitute.hellbender.utils.gvs.bigquery.BigQueryUtils;
 
 import java.io.File;
 import java.io.IOException;

diff --git a/...main/java/org/broadinstitute/hellbender/tools/gvs/ingest/VcfHeaderLineScratchCreator.java b/...main/java/org/broadinstitute/hellbender/tools/gvs/ingest/VcfHeaderLineScratchCreator.java
@@ -3,12 +3,11 @@
 import com.google.protobuf.Descriptors;
 import org.broadinstitute.hellbender.exceptions.UserException;
 import org.broadinstitute.hellbender.utils.Utils;
-import org.broadinstitute.hellbender.utils.bigquery.BigQueryUtils;
+import org.broadinstitute.hellbender.utils.gvs.bigquery.BigQueryUtils;
 import org.broadinstitute.hellbender.utils.gvs.bigquery.PendingBQWriter;
 import org.json.JSONObject;
 
 import java.io.IOException;
-import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
 

diff --git a/src/main/java/org/broadinstitute/hellbender/tools/gvs/ingest/VetCreator.java b/src/main/java/org/broadinstitute/hellbender/tools/gvs/ingest/VetCreator.java
@@ -7,7 +7,7 @@
 import org.broadinstitute.hellbender.tools.gvs.common.CommonCode;
 import org.broadinstitute.hellbender.tools.gvs.common.IngestConstants;
 import org.broadinstitute.hellbender.tools.gvs.common.SchemaUtils;
-import org.broadinstitute.hellbender.utils.bigquery.BigQueryUtils;
+import org.broadinstitute.hellbender.utils.gvs.bigquery.BigQueryUtils;
 import org.broadinstitute.hellbender.utils.gvs.bigquery.PendingBQWriter;
 import org.broadinstitute.hellbender.utils.tsv.SimpleXSVWriter;
 import org.json.JSONObject;

diff --git a/src/main/java/org/broadinstitute/hellbender/utils/gvs/bigquery/AvroFileReader.java b/src/main/java/org/broadinstitute/hellbender/utils/gvs/bigquery/AvroFileReader.java
@@ -7,7 +7,6 @@
 import org.apache.avro.io.DatumReader;
 import org.broadinstitute.hellbender.engine.GATKPath;
 import org.broadinstitute.hellbender.exceptions.GATKException;
-import org.broadinstitute.hellbender.utils.bigquery.GATKAvroReader;
 
 import java.io.IOException;
 import java.util.Iterator;

diff --git a/...bigquery/BigQueryResultAndStatistics.java → ...bigquery/BigQueryResultAndStatistics.java b/...bigquery/BigQueryResultAndStatistics.java → ...bigquery/BigQueryResultAndStatistics.java
@@ -1,4 +1,4 @@
-package org.broadinstitute.hellbender.utils.bigquery;
+package org.broadinstitute.hellbender.utils.gvs.bigquery;
 
 import com.google.cloud.bigquery.JobStatistics;
 import com.google.cloud.bigquery.TableResult;

diff --git a/...lbender/utils/bigquery/BigQueryUtils.java → ...der/utils/gvs/bigquery/BigQueryUtils.java b/...lbender/utils/bigquery/BigQueryUtils.java → ...der/utils/gvs/bigquery/BigQueryUtils.java
@@ -1,4 +1,4 @@
-package org.broadinstitute.hellbender.utils.bigquery;
+package org.broadinstitute.hellbender.utils.gvs.bigquery;
 
 import com.google.cloud.bigquery.*;
 import io.grpc.StatusRuntimeException;
@@ -415,12 +415,12 @@ private static long getQueryCostBytesProcessedEstimate(String queryString, Strin
         return bytesProcessed;
     }
     public static StorageAPIAvroReaderAndBigQueryStatistics executeQueryWithStorageAPI(final String queryString,
-                                                                  final List<String> fieldsToRetrieve,
-                                                                  final String projectID,
-                                                                  final String datasetID,
-                                                                  final String userDefinedFunctions,
-                                                                  final boolean runQueryInBatchMode,
-                                                                  Map<String, String> labels) {
+                                                                                       final List<String> fieldsToRetrieve,
+                                                                                       final String projectID,
+                                                                                       final String datasetID,
+                                                                                       final String userDefinedFunctions,
+                                                                                       final boolean runQueryInBatchMode,
+                                                                                       Map<String, String> labels) {
         final String tempTableName = String.format("%s_%s", "temp_table", UUID.randomUUID().toString().replace('-', '_'));
         final String tempTableFullyQualified = String.format("%s.%s.%s", projectID, datasetID, tempTableName);
 

diff --git a/src/main/java/org/broadinstitute/hellbender/utils/gvs/bigquery/CommittedBQWriter.java b/src/main/java/org/broadinstitute/hellbender/utils/gvs/bigquery/CommittedBQWriter.java
@@ -15,7 +15,7 @@
 import java.io.IOException;
 import java.util.concurrent.ExecutionException;
 
-import static org.broadinstitute.hellbender.utils.bigquery.BigQueryUtils.extractCausalStatusRuntimeExceptionOrThrow;
+import static org.broadinstitute.hellbender.utils.gvs.bigquery.BigQueryUtils.extractCausalStatusRuntimeExceptionOrThrow;
 
 
 public class CommittedBQWriter implements AutoCloseable {

diff --git a/...bender/utils/bigquery/GATKAvroReader.java → ...er/utils/gvs/bigquery/GATKAvroReader.java b/...bender/utils/bigquery/GATKAvroReader.java → ...er/utils/gvs/bigquery/GATKAvroReader.java
@@ -1,8 +1,8 @@
-package org.broadinstitute.hellbender.utils.bigquery;
+package org.broadinstitute.hellbender.utils.gvs.bigquery;
 
 import htsjdk.samtools.util.CloseableIterator;
 import org.apache.avro.generic.GenericRecord;
 
 public interface GATKAvroReader extends Iterable<GenericRecord>, CloseableIterator<GenericRecord> {
     org.apache.avro.Schema getSchema();
-}
+}
diff --git a/.../utils/bigquery/StorageAPIAvroReader.java → ...ls/gvs/bigquery/StorageAPIAvroReader.java b/.../utils/bigquery/StorageAPIAvroReader.java → ...ls/gvs/bigquery/StorageAPIAvroReader.java
@@ -1,4 +1,4 @@
-package org.broadinstitute.hellbender.utils.bigquery;
+package org.broadinstitute.hellbender.utils.gvs.bigquery;
 
 import com.google.cloud.bigquery.storage.v1.AvroRows;
 import com.google.cloud.bigquery.storage.v1.BigQueryReadClient;

diff --git a/...geAPIAvroReaderAndBigQueryStatistics.java → ...geAPIAvroReaderAndBigQueryStatistics.java b/...geAPIAvroReaderAndBigQueryStatistics.java → ...geAPIAvroReaderAndBigQueryStatistics.java
@@ -1,4 +1,4 @@
-package org.broadinstitute.hellbender.utils.bigquery;
+package org.broadinstitute.hellbender.utils.gvs.bigquery;
 
 import com.google.cloud.bigquery.JobStatistics;
 

diff --git a/...bender/utils/bigquery/TableReference.java → ...er/utils/gvs/bigquery/TableReference.java b/...bender/utils/bigquery/TableReference.java → ...er/utils/gvs/bigquery/TableReference.java
@@ -1,4 +1,4 @@
-package org.broadinstitute.hellbender.utils.bigquery;
+package org.broadinstitute.hellbender.utils.gvs.bigquery;
 
 import com.google.common.collect.ImmutableList;
 

diff --git a/...utils/bigquery/BigQueryUtilsUnitTest.java → ...s/gvs/bigquery/BigQueryUtilsUnitTest.java b/...utils/bigquery/BigQueryUtilsUnitTest.java → ...s/gvs/bigquery/BigQueryUtilsUnitTest.java
@@ -1,4 +1,4 @@
-package org.broadinstitute.hellbender.utils.bigquery;
+package org.broadinstitute.hellbender.utils.gvs.bigquery;
 
 import com.google.cloud.bigquery.FieldValueList;
 import org.apache.avro.generic.GenericRecord;

diff --git a/src/test/java/org/broadinstitute/hellbender/utils/gvs/bigquery/LoadStatusBQTest.java b/src/test/java/org/broadinstitute/hellbender/utils/gvs/bigquery/LoadStatusBQTest.java
@@ -3,7 +3,6 @@
 import org.broadinstitute.hellbender.GATKBaseTest;
 import org.broadinstitute.hellbender.tools.gvs.common.SchemaUtils;
 import org.broadinstitute.hellbender.tools.gvs.ingest.LoadStatus;
-import org.broadinstitute.hellbender.utils.bigquery.BigQueryUtils;
 import org.testng.Assert;
 import org.testng.annotations.AfterTest;
 import org.testng.annotations.BeforeTest;

diff --git a/src/test/java/org/broadinstitute/hellbender/utils/gvs/bigquery/PendingBQWriterTest.java b/src/test/java/org/broadinstitute/hellbender/utils/gvs/bigquery/PendingBQWriterTest.java
@@ -3,8 +3,6 @@
 import com.google.cloud.bigquery.FieldValueList;
 import com.google.protobuf.Descriptors;
 import org.broadinstitute.hellbender.GATKBaseTest;
-import org.broadinstitute.hellbender.utils.bigquery.BigQueryResultAndStatistics;
-import org.broadinstitute.hellbender.utils.bigquery.BigQueryUtils;
 import org.json.JSONObject;
 import org.testng.Assert;
 import org.testng.annotations.AfterTest;