Merge pull request #221 from umccr/build/deps

build: update noodles and other dependencies

htsget-actix/src/lib.rs

@@ -201,8 +201,7 @@ mod tests {
   #[async_trait(?Send)]
   impl TestServer<ActixTestRequest<test::TestRequest>> for ActixTestServer {
     async fn get_expected_path(&self) -> String {
-      let mut bind_data_server =
-        BindDataServer::try_from(self.get_config().data_server().clone()).unwrap();
+      let mut bind_data_server = BindDataServer::from(self.get_config().data_server().clone());
       let server = bind_data_server.bind_data_server().await.unwrap();
       let addr = server.local_addr();
 

htsget-config/Cargo.toml

@@ -18,7 +18,7 @@ default = []
 [dependencies]
 thiserror = "1.0"
 async-trait = "0.1"
-noodles = { version = "0.50", features = ["core"] }
+noodles = { version = "0.60", features = ["core"] }
 serde = { version = "1.0", features = ["derive"] }
 serde_with = "3.0"
 serde_regex = "1.1"
@@ -27,7 +27,7 @@ figment = { version = "0.10", features = ["env", "toml"] }
 clap = { version = "4.3", features = ["derive", "env", "cargo"] }
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["registry", "env-filter", "ansi", "json"] }
-toml = "0.7"
+toml = "0.8"
 http = "0.2"
 http-serde = "1.1"
 rustls-pemfile = "1.0"
@@ -42,4 +42,4 @@ serde_json = "1.0"
 figment = { version = "0.10", features = ["test"] }
 tokio = { version = "1.28", features = ["macros", "rt-multi-thread"] }
 tempfile = "3.6"
-rcgen = "0.11"
+rcgen = "0.12"

htsget-lambda/Cargo.toml

@@ -34,5 +34,5 @@ bytes = "1.4"
 
 [dev-dependencies]
 async-trait = "0.1"
-query_map = { version = "0.6", features = ["url-query"] }
+query_map = { version = "0.7", features = ["url-query"] }
 tempfile = "3.6"

htsget-lambda/src/lib.rs

@@ -708,7 +708,7 @@ mod tests {
   }
 
   async fn spawn_server(config: &Config) -> String {
-    let mut bind_data_server = BindDataServer::try_from(config.data_server().clone()).unwrap();
+    let mut bind_data_server = BindDataServer::from(config.data_server().clone());
     let server = bind_data_server.bind_data_server().await.unwrap();
     let addr = server.local_addr();
 

htsget-search/Cargo.toml

@@ -47,11 +47,11 @@ futures-util = "0.3"
 async-trait = "0.1"
 
 # Noodles
-noodles = { version = "0.50", features = ["async", "core", "bgzf", "bam", "bcf", "cram", "csi", "sam", "tabix", "vcf"] }
+noodles = { version = "0.60", features = ["async", "core", "bgzf", "bam", "bcf", "cram", "csi", "sam", "tabix", "vcf"] }
 
 # Amazon S3
 bytes = { version = "1.4", optional = true }
-aws-sdk-s3 = { version = "0.30", optional = true }
+aws-sdk-s3 = { version = "0.34", optional = true }
 aws-config = { version = "0.56", optional = true }
 
 # Url storage

htsget-search/src/htsget/bam_search.rs

@@ -7,10 +7,12 @@ use std::sync::Arc;
 use async_trait::async_trait;
 use noodles::bam;
 use noodles::bam::bai;
+use noodles::bam::bai::Index;
 use noodles::bgzf;
 use noodles::bgzf::VirtualPosition;
-use noodles::csi::index::ReferenceSequence;
-use noodles::csi::Index;
+use noodles::csi::binning_index::index::reference_sequence::index::LinearIndex;
+use noodles::csi::binning_index::index::ReferenceSequence;
+use noodles::csi::BinningIndex;
 use noodles::sam::header::record::value::map::read_group::platform::ParseError;
 use noodles::sam::header::record::value::map::read_group::Platform;
 use noodles::sam::Header;
@@ -35,7 +37,8 @@ pub struct BamSearch<S> {
 }
 
 #[async_trait]
-impl<S, ReaderType> BgzfSearch<S, ReaderType, AsyncReader<ReaderType>, Header> for BamSearch<S>
+impl<S, ReaderType> BgzfSearch<S, LinearIndex, ReaderType, AsyncReader<ReaderType>, Header>
+  for BamSearch<S>
 where
   S: Storage<Streamable = ReaderType> + Send + Sync + 'static,
   ReaderType: AsyncRead + Unpin + Send + Sync,
@@ -47,7 +50,7 @@ where
     index: &Index,
   ) -> Result<Vec<BytesPosition>> {
     trace!("getting byte ranges for unmapped reads");
-    let last_interval = index.first_record_in_last_linear_bin_start_position();
+    let last_interval = index.last_first_record_start_position();
     let start = match last_interval {
       Some(start) => start,
       None => {
@@ -78,7 +81,8 @@ where
 }
 
 #[async_trait]
-impl<S, ReaderType> Search<S, ReaderType, ReferenceSequence, Index, AsyncReader<ReaderType>, Header>
+impl<S, ReaderType>
+  Search<S, ReaderType, ReferenceSequence<LinearIndex>, Index, AsyncReader<ReaderType>, Header>
   for BamSearch<S>
 where
   S: Storage<Streamable = ReaderType> + Send + Sync + 'static,
@@ -121,7 +125,6 @@ where
 
   async fn read_index_inner<T: AsyncRead + Unpin + Send>(inner: T) -> io::Result<Index> {
     let mut reader = bai::AsyncReader::new(BufReader::new(inner));
-    reader.read_header().await?;
     reader.read_index().await
   }
 
@@ -150,7 +153,7 @@ where
 
 #[async_trait]
 impl<S, ReaderType>
-  SearchReads<S, ReaderType, ReferenceSequence, Index, AsyncReader<ReaderType>, Header>
+  SearchReads<S, ReaderType, ReferenceSequence<LinearIndex>, Index, AsyncReader<ReaderType>, Header>
   for BamSearch<S>
 where
   S: Storage<Streamable = ReaderType> + Send + Sync + 'static,

htsget-search/src/htsget/bcf_search.rs

@@ -7,7 +7,8 @@ use async_trait::async_trait;
 use futures_util::stream::FuturesOrdered;
 use noodles::bcf;
 use noodles::bgzf::VirtualPosition;
-use noodles::csi::index::ReferenceSequence;
+use noodles::csi::binning_index::index::reference_sequence::index::BinnedIndex;
+use noodles::csi::binning_index::index::ReferenceSequence;
 use noodles::csi::Index;
 use noodles::vcf::Header;
 use noodles::{bgzf, csi};
@@ -28,7 +29,8 @@ pub struct BcfSearch<S> {
 }
 
 #[async_trait]
-impl<S, ReaderType> BgzfSearch<S, ReaderType, AsyncReader<ReaderType>, Header> for BcfSearch<S>
+impl<S, ReaderType> BgzfSearch<S, BinnedIndex, ReaderType, AsyncReader<ReaderType>, Header>
+  for BcfSearch<S>
 where
   S: Storage<Streamable = ReaderType> + Send + Sync + 'static,
   ReaderType: AsyncRead + Unpin + Send + Sync,
@@ -43,7 +45,8 @@ where
 }
 
 #[async_trait]
-impl<S, ReaderType> Search<S, ReaderType, ReferenceSequence, Index, AsyncReader<ReaderType>, Header>
+impl<S, ReaderType>
+  Search<S, ReaderType, ReferenceSequence<BinnedIndex>, Index, AsyncReader<ReaderType>, Header>
   for BcfSearch<S>
 where
   S: Storage<Streamable = ReaderType> + Send + Sync + 'static,

htsget-search/src/htsget/search.rs

@@ -12,9 +12,11 @@ use async_trait::async_trait;
 use futures::StreamExt;
 use futures_util::stream::FuturesOrdered;
 use noodles::bgzf::{gzi, VirtualPosition};
-use noodles::csi::index::reference_sequence::bin::Chunk;
-use noodles::csi::index::ReferenceSequence;
-use noodles::csi::Index;
+use noodles::csi::binning_index::index::reference_sequence::bin::Chunk;
+use noodles::csi::binning_index::index::Index;
+use noodles::csi::binning_index::index::{reference_sequence, ReferenceSequence};
+use noodles::csi::binning_index::ReferenceSequence as ReferenceSequenceExt;
+use noodles::csi::BinningIndex;
 use tokio::io;
 use tokio::io::{AsyncRead, BufReader};
 use tokio::select;
@@ -373,22 +375,24 @@ where
 /// BAM, BCF, and VCF.
 ///
 /// [S] is the storage type.
+/// [I] the index type used for the `ReferenceSequence`.
 /// [ReaderType] is the inner type used for [Reader].
 /// [ReferenceSequence] is the reference sequence type of the format's index.
 /// [Index] is the format's index type.
 /// [Reader] is the format's reader type.
 /// [Header] is the format's header type.
 #[async_trait]
-pub trait BgzfSearch<S, ReaderType, Reader, Header>:
-  Search<S, ReaderType, ReferenceSequence, Index, Reader, Header>
+pub trait BgzfSearch<S, I, ReaderType, Reader, Header>:
+  Search<S, ReaderType, ReferenceSequence<I>, Index<I>, Reader, Header>
 where
   S: Storage<Streamable = ReaderType> + Send + Sync + 'static,
+  I: reference_sequence::Index + Send + Sync,
   ReaderType: AsyncRead + Unpin + Send + Sync,
   Reader: Send + Sync,
   Header: Send + Sync,
 {
   #[instrument(level = "trace", skip_all)]
-  fn index_positions(index: &Index) -> BTreeSet<u64> {
+  fn index_positions(index: &Index<I>) -> BTreeSet<u64> {
     trace!("getting possible index positions");
     let mut positions = BTreeSet::new();
 
@@ -425,7 +429,7 @@ where
     &self,
     query: &Query,
     ref_seq_id: usize,
-    index: &Index,
+    index: &Index<I>,
   ) -> Result<Vec<BytesPosition>> {
     let chunks: Result<Vec<Chunk>> = trace_span!("querying chunks").in_scope(|| {
       trace!(id = ?query.id(), ref_seq_id = ?ref_seq_id, "querying chunks");
@@ -538,7 +542,7 @@ where
   async fn get_byte_ranges_for_unmapped(
     &self,
     _query: &Query,
-    _index: &Index,
+    _index: &Index<I>,
   ) -> Result<Vec<BytesPosition>> {
     Ok(Vec::new())
   }
@@ -551,14 +555,15 @@ where
 }
 
 #[async_trait]
-impl<S, ReaderType, Reader, Header, T>
-  SearchAll<S, ReaderType, ReferenceSequence, Index, Reader, Header> for T
+impl<S, I, ReaderType, Reader, Header, T>
+  SearchAll<S, ReaderType, ReferenceSequence<I>, Index<I>, Reader, Header> for T
 where
   S: Storage<Streamable = ReaderType> + Send + Sync + 'static,
+  I: reference_sequence::Index + Send + Sync,
   ReaderType: AsyncRead + Unpin + Send + Sync,
   Reader: Send + Sync,
   Header: Send + Sync,
-  T: BgzfSearch<S, ReaderType, Reader, Header> + Send + Sync,
+  T: BgzfSearch<S, I, ReaderType, Reader, Header> + Send + Sync,
 {
   #[instrument(level = "debug", skip(self), ret)]
   async fn get_byte_ranges_for_all(&self, query: &Query) -> Result<Vec<BytesPosition>> {
@@ -568,7 +573,7 @@ where
   }
 
   #[instrument(level = "trace", skip_all, ret)]
-  async fn get_header_end_offset(&self, index: &Index) -> Result<u64> {
+  async fn get_header_end_offset(&self, index: &Index<I>) -> Result<u64> {
     let first_index_position =
       Self::index_positions(index)
         .into_iter()
@@ -587,7 +592,7 @@ where
 
   async fn get_byte_ranges_for_header(
     &self,
-    index: &Index,
+    index: &Index<I>,
     header: &Header,
     reader: &mut Reader,
     query: &Query,

htsget-search/src/htsget/vcf_search.rs

@@ -7,9 +7,11 @@ use async_trait::async_trait;
 use futures_util::stream::FuturesOrdered;
 use noodles::bgzf;
 use noodles::bgzf::VirtualPosition;
-use noodles::csi::index::ReferenceSequence;
-use noodles::csi::Index;
+use noodles::csi::binning_index::index::reference_sequence::index::LinearIndex;
+use noodles::csi::binning_index::index::ReferenceSequence;
+use noodles::csi::BinningIndex;
 use noodles::tabix;
+use noodles::tabix::Index;
 use noodles::vcf;
 use noodles::vcf::Header;
 use tokio::io;
@@ -30,7 +32,8 @@ pub struct VcfSearch<S> {
 }
 
 #[async_trait]
-impl<S, ReaderType> BgzfSearch<S, ReaderType, AsyncReader<ReaderType>, Header> for VcfSearch<S>
+impl<S, ReaderType> BgzfSearch<S, LinearIndex, ReaderType, AsyncReader<ReaderType>, Header>
+  for VcfSearch<S>
 where
   S: Storage<Streamable = ReaderType> + Send + Sync + 'static,
   ReaderType: AsyncRead + Unpin + Send + Sync,
@@ -48,7 +51,8 @@ where
 }
 
 #[async_trait]
-impl<S, ReaderType> Search<S, ReaderType, ReferenceSequence, Index, AsyncReader<ReaderType>, Header>
+impl<S, ReaderType>
+  Search<S, ReaderType, ReferenceSequence<LinearIndex>, Index, AsyncReader<ReaderType>, Header>
   for VcfSearch<S>
 where
   S: Storage<Streamable = ReaderType> + Send + Sync + 'static,

htsget-test/Cargo.toml

@@ -47,7 +47,7 @@ default = []
 # Server tests dependencies
 htsget-config = { version = "0.8.0", path = "../htsget-config", default-features = false, optional = true }
 
-noodles = { version = "0.50", optional = true, features = ["async", "bgzf", "vcf"] }
+noodles = { version = "0.60", optional = true, features = ["async", "bgzf", "vcf"] }
 
 reqwest = { version = "0.11", default-features = false, features = ["json", "rustls-tls"], optional = true }
 tokio = { version = "1", features = ["rt-multi-thread"], optional = true }
@@ -60,12 +60,12 @@ serde = { version = "1", optional = true }
 base64 = { version = "0.21", optional = true }
 
 tempfile = { version = "3.3", optional = true }
-aws-sdk-s3 = { version = "0.30", features = ["test-util"], optional = true }
+aws-sdk-s3 = { version = "0.34", features = ["test-util"], optional = true }
 aws-config = { version = "0.56", optional = true }
 aws-credential-types = { version = "0.56", features = ["test-util"], optional = true }
-s3s = { version = "0.7", optional = true }
-s3s-fs = { version = "0.7", optional = true }
-s3s-aws = { version = "0.7", optional = true }
+s3s = { version = "0.8", optional = true }
+s3s-fs = { version = "0.8", optional = true }
+s3s-aws = { version = "0.8", optional = true }
 
 # Default dependencies
-rcgen = "0.11"
+rcgen = "0.12"