Merge pull request #1183 from milaboratory/tiny-things

Tiny things
milaboratory · Jun 30, 2023 · b0128a7 · b0128a7
2 parents 6e0656a + baf4656
commit b0128a7
Show file tree

Hide file tree

Showing 191 changed files with 426 additions and 74 deletions.
diff --git a/build.gradle.kts b/build.gradle.kts
@@ -130,7 +130,7 @@ val toObfuscate: Configuration by configurations.creating {
 
 val obfuscationLibs: Configuration by configurations.creating
 
-val mixcrAlgoVersion = "4.3.0-208-develop"
+val mixcrAlgoVersion = "4.3.0-215-tiny-things"
 val milibVersion = ""
 val mitoolVersion = ""
 val repseqioVersion = ""

diff --git a/changelogs/v4.4.md b/changelogs/v4.4.md
@@ -131,17 +131,31 @@
 - added gene feature coverage in alignment report
 - on Linux platforms default calculation of -Xmx now based on "available" memory (previously "free" was used)
 - new gene aligner parameter `edgeRealignmentMinScoreOverride` for more sensitive alignments for short paired-end reads
-- report values downstream `align` now calculate percents relative to the number of reads in the sample rather than the total number of reads in multi-sample analysis
-- options helping with advanced analysis of data quality and consensus assembly process added to `assemble` (`--consensus-alignments`, `--consensus-state-stat`, `--downsample-consensus-state-stat`) and `analyze` (`--output-consensus-alignments`, `--output-consensus-state-stat`, `--downsample-consensus-state-stat`)
+- report values downstream `align` now calculate percents relative to the number of reads in the sample rather than the
+  total number of reads in multi-sample analysis
+- options helping with advanced analysis of data quality and consensus assembly process added
+  to `assemble` (`--consensus-alignments`, `--consensus-state-stat`, `--downsample-consensus-state-stat`)
+  and `analyze` (`--output-consensus-alignments`, `--output-consensus-state-stat`, `--downsample-consensus-state-stat`)
 - fixed bug not allowing to parse more than two reads with tag pattern
 - better tag pattern search projection representation in reports
 - fix for value in report line `Reads dropped due to low quality, percent of total report string`
 - additional report string "Aligned reads processed" in `assemble` report
-- fix bug when `--chains` is used with `exportClonesOverlap` 
+- fix bug when `--chains` is used with `exportClonesOverlap`
 - fix for `export...` - tag quality field added back to export columns
 - fixes bug not allowing to use more than 2 input reads in pattern matching
-- add options `--by-feature` and `--by-gene` to `sortClones`
+- added options `--by-feature` and `--by-gene` to `sortClones`
+- added options `-rankByReads` and `-rankByTag <(Molecule|Cell|Sample)>` to `exportClones`
+  and `exportShmTreesWithNodes.txt`
+- export readIds in `exportAlignments` by default
+- `findAlleles` now recalculate functionality of de novo found alleles
+- add info about CDR3 in generated hash for de novo alleles
+- more robust algorithm for calculating checksum of library
+- remove de novo alleles that are actually the same, will be left only one
+- `findAlleles` will remove not used genes from library (genes that not represented in given donor)
 - make `--chains` optional in `downsampling` command and allow multiple input
+- write empty file on `exportClones` if file doesn't contain any clones
+- better exception messages on incorrect inputs for export commands
+- on `exportClones` write `no_d_gene` if requested VDJunction, DCDR3Part or DJJunction in absence of D hit
 
 ## Braking changes
 

diff --git a/regression/presets/analyze/10x-5gex-cdr3.yaml b/regression/presets/analyze/10x-5gex-cdr3.yaml
@@ -328,6 +328,7 @@ exportAlignments:
   chains: ALL
   noHeader: false
   fields:
+  - field: -readIds
   - field: -targetSequences
   - field: -targetQualities
   - field: -vHitsWithScore

diff --git a/regression/presets/analyze/10x-5gex-full-length.yaml b/regression/presets/analyze/10x-5gex-full-length.yaml
@@ -332,6 +332,7 @@ exportAlignments:
   chains: ALL
   noHeader: false
   fields:
+  - field: -readIds
   - field: -targetSequences
   - field: -targetQualities
   - field: -vHitsWithScore

diff --git a/regression/presets/analyze/10x-5gex.yaml b/regression/presets/analyze/10x-5gex.yaml
@@ -328,6 +328,7 @@ exportAlignments:
   chains: ALL
   noHeader: false
   fields:
+  - field: -readIds
   - field: -targetSequences
   - field: -targetQualities
   - field: -vHitsWithScore

diff --git a/regression/presets/analyze/10x-vdj-bcr-full-length.yaml b/regression/presets/analyze/10x-vdj-bcr-full-length.yaml
@@ -500,6 +500,7 @@ exportAlignments:
   chains: ALL
   noHeader: false
   fields:
+  - field: -readIds
   - field: -targetSequences
   - field: -targetQualities
   - field: -vHitsWithScore

diff --git a/regression/presets/analyze/10x-vdj-bcr.yaml b/regression/presets/analyze/10x-vdj-bcr.yaml
@@ -496,6 +496,7 @@ exportAlignments:
   chains: ALL
   noHeader: false
   fields:
+  - field: -readIds
   - field: -targetSequences
   - field: -targetQualities
   - field: -vHitsWithScore

diff --git a/regression/presets/analyze/10x-vdj-tcr-alias-multi-barcode-test.yaml b/regression/presets/analyze/10x-vdj-tcr-alias-multi-barcode-test.yaml
@@ -463,6 +463,7 @@ exportAlignments:
   chains: ALL
   noHeader: false
   fields:
+  - field: -readIds
   - field: -targetSequences
   - field: -targetQualities
   - field: -vHitsWithScore

diff --git a/regression/presets/analyze/10x-vdj-tcr-full-length.yaml b/regression/presets/analyze/10x-vdj-tcr-full-length.yaml
@@ -500,6 +500,7 @@ exportAlignments:
   chains: ALL
   noHeader: false
   fields:
+  - field: -readIds
   - field: -targetSequences
   - field: -targetQualities
   - field: -vHitsWithScore

diff --git a/regression/presets/analyze/10x-vdj-tcr-multi-barcode-test.yaml b/regression/presets/analyze/10x-vdj-tcr-multi-barcode-test.yaml
@@ -426,6 +426,7 @@ exportAlignments:
   chains: ALL
   noHeader: false
   fields:
+  - field: -readIds
   - field: -targetSequences
   - field: -targetQualities
   - field: -vHitsWithScore

diff --git a/regression/presets/analyze/10x-vdj-tcr-qc-test.yaml b/regression/presets/analyze/10x-vdj-tcr-qc-test.yaml
@@ -499,6 +499,7 @@ exportAlignments:
   chains: ALL
   noHeader: false
   fields:
+  - field: -readIds
   - field: -targetSequences
   - field: -targetQualities
   - field: -vHitsWithScore

diff --git a/regression/presets/analyze/10x-vdj-tcr.yaml b/regression/presets/analyze/10x-vdj-tcr.yaml
@@ -496,6 +496,7 @@ exportAlignments:
   chains: ALL
   noHeader: false
   fields:
+  - field: -readIds
   - field: -targetSequences
   - field: -targetQualities
   - field: -vHitsWithScore

diff --git a/regression/presets/analyze/10x-vdj.yaml b/regression/presets/analyze/10x-vdj.yaml
@@ -496,6 +496,7 @@ exportAlignments:
   chains: ALL
   noHeader: false
   fields:
+  - field: -readIds
   - field: -targetSequences
   - field: -targetQualities
   - field: -vHitsWithScore

diff --git a/regression/presets/analyze/abhelix-human-bcr-cdr3.yaml b/regression/presets/analyze/abhelix-human-bcr-cdr3.yaml
@@ -267,6 +267,7 @@ exportAlignments:
   chains: ALL
   noHeader: false
   fields:
+  - field: -readIds
   - field: -targetSequences
   - field: -targetQualities
   - field: -vHitsWithScore

diff --git a/regression/presets/analyze/abhelix-human-bcr-full-length.yaml b/regression/presets/analyze/abhelix-human-bcr-full-length.yaml
@@ -267,6 +267,7 @@ exportAlignments:
   chains: ALL
   noHeader: false
   fields:
+  - field: -readIds
   - field: -targetSequences
   - field: -targetQualities
   - field: -vHitsWithScore

diff --git a/regression/presets/analyze/abhelix-human-tcr-cdr3.yaml b/regression/presets/analyze/abhelix-human-tcr-cdr3.yaml
@@ -267,6 +267,7 @@ exportAlignments:
   chains: ALL
   noHeader: false
   fields:
+  - field: -readIds
   - field: -targetSequences
   - field: -targetQualities
   - field: -vHitsWithScore

diff --git a/regression/presets/analyze/abhelix-human-tcr-full-length.yaml b/regression/presets/analyze/abhelix-human-tcr-full-length.yaml
@@ -267,6 +267,7 @@ exportAlignments:
   chains: ALL
   noHeader: false
   fields:
+  - field: -readIds
   - field: -targetSequences
   - field: -targetQualities
   - field: -vHitsWithScore

diff --git a/regression/presets/analyze/abhelix-human.yaml b/regression/presets/analyze/abhelix-human.yaml
@@ -267,6 +267,7 @@ exportAlignments:
   chains: ALL
   noHeader: false
   fields:
+  - field: -readIds
   - field: -targetSequences
   - field: -targetQualities
   - field: -vHitsWithScore

diff --git a/regression/presets/analyze/ampliseq-tcrb-plus-cdr3.yaml b/regression/presets/analyze/ampliseq-tcrb-plus-cdr3.yaml
@@ -240,6 +240,7 @@ exportAlignments:
   chains: ALL
   noHeader: false
   fields:
+  - field: -readIds
   - field: -targetSequences
   - field: -targetQualities
   - field: -vHitsWithScore

diff --git a/regression/presets/analyze/ampliseq-tcrb-plus-full-length.yaml b/regression/presets/analyze/ampliseq-tcrb-plus-full-length.yaml
@@ -240,6 +240,7 @@ exportAlignments:
   chains: ALL
   noHeader: false
   fields:
+  - field: -readIds
   - field: -targetSequences
   - field: -targetQualities
   - field: -vHitsWithScore

diff --git a/regression/presets/analyze/ampliseq-tcrb-plus.yaml b/regression/presets/analyze/ampliseq-tcrb-plus.yaml
@@ -240,6 +240,7 @@ exportAlignments:
   chains: ALL
   noHeader: false
   fields:
+  - field: -readIds
   - field: -targetSequences
   - field: -targetQualities
   - field: -vHitsWithScore

diff --git a/regression/presets/analyze/ampliseq-tcrb-sr-cdr3.yaml b/regression/presets/analyze/ampliseq-tcrb-sr-cdr3.yaml
@@ -240,6 +240,7 @@ exportAlignments:
   chains: ALL
   noHeader: false
   fields:
+  - field: -readIds
   - field: -targetSequences
   - field: -targetQualities
   - field: -vHitsWithScore

diff --git a/regression/presets/analyze/ampliseq-tcrb-sr.yaml b/regression/presets/analyze/ampliseq-tcrb-sr.yaml
@@ -240,6 +240,7 @@ exportAlignments:
   chains: ALL
   noHeader: false
   fields:
+  - field: -readIds
   - field: -targetSequences
   - field: -targetQualities
   - field: -vHitsWithScore

diff --git a/regression/presets/analyze/bd-rhapsody-human-bcr-full-length.yaml b/regression/presets/analyze/bd-rhapsody-human-bcr-full-length.yaml
@@ -388,6 +388,7 @@ exportAlignments:
   chains: ALL
   noHeader: false
   fields:
+  - field: -readIds
   - field: -targetSequences
   - field: -targetQualities
   - field: -vHitsWithScore

diff --git a/regression/presets/analyze/bd-rhapsody-human-bcr-v1.yaml b/regression/presets/analyze/bd-rhapsody-human-bcr-v1.yaml
@@ -420,7 +420,24 @@ assemble:
       requiredSequences: []
   inferMinRecordsPerConsensus: true
 assembleContigs: null
-exportAlignments: null
+exportAlignments:
+  chains: ALL
+  noHeader: false
+  fields:
+  - field: -readIds
+  - field: -targetSequences
+  - field: -targetQualities
+  - field: -vHitsWithScore
+  - field: -dHitsWithScore
+  - field: -jHitsWithScore
+  - field: -cHitsWithScore
+  - field: -vAlignments
+  - field: -dAlignments
+  - field: -jAlignments
+  - field: -cAlignments
+  - field: -allNFeaturesWithMinQuality
+  - field: -allAAFeatures
+  - field: -defaultAnchorPoints
 exportClones:
   splitByTagType: null
   filterOutOfFrames: false

diff --git a/regression/presets/analyze/bd-rhapsody-human-bcr-v2.yaml b/regression/presets/analyze/bd-rhapsody-human-bcr-v2.yaml
@@ -423,7 +423,24 @@ assemble:
       requiredSequences: []
   inferMinRecordsPerConsensus: true
 assembleContigs: null
-exportAlignments: null
+exportAlignments:
+  chains: ALL
+  noHeader: false
+  fields:
+  - field: -readIds
+  - field: -targetSequences
+  - field: -targetQualities
+  - field: -vHitsWithScore
+  - field: -dHitsWithScore
+  - field: -jHitsWithScore
+  - field: -cHitsWithScore
+  - field: -vAlignments
+  - field: -dAlignments
+  - field: -jAlignments
+  - field: -cAlignments
+  - field: -allNFeaturesWithMinQuality
+  - field: -allAAFeatures
+  - field: -defaultAnchorPoints
 exportClones:
   splitByTagType: null
   filterOutOfFrames: false

diff --git a/regression/presets/analyze/bd-rhapsody-human-tcr-full-length.yaml b/regression/presets/analyze/bd-rhapsody-human-tcr-full-length.yaml
@@ -367,6 +367,7 @@ exportAlignments:
   chains: ALL
   noHeader: false
   fields:
+  - field: -readIds
   - field: -targetSequences
   - field: -targetQualities
   - field: -vHitsWithScore

diff --git a/regression/presets/analyze/bd-rhapsody-human-tcr-v1.yaml b/regression/presets/analyze/bd-rhapsody-human-tcr-v1.yaml
@@ -417,7 +417,24 @@ assemble:
       requiredSequences: []
   inferMinRecordsPerConsensus: true
 assembleContigs: null
-exportAlignments: null
+exportAlignments:
+  chains: ALL
+  noHeader: false
+  fields:
+  - field: -readIds
+  - field: -targetSequences
+  - field: -targetQualities
+  - field: -vHitsWithScore
+  - field: -dHitsWithScore
+  - field: -jHitsWithScore
+  - field: -cHitsWithScore
+  - field: -vAlignments
+  - field: -dAlignments
+  - field: -jAlignments
+  - field: -cAlignments
+  - field: -allNFeaturesWithMinQuality
+  - field: -allAAFeatures
+  - field: -defaultAnchorPoints
 exportClones:
   splitByTagType: null
   filterOutOfFrames: false

diff --git a/regression/presets/analyze/bd-rhapsody-human-tcr-v2.yaml b/regression/presets/analyze/bd-rhapsody-human-tcr-v2.yaml
@@ -420,7 +420,24 @@ assemble:
       requiredSequences: []
   inferMinRecordsPerConsensus: true
 assembleContigs: null
-exportAlignments: null
+exportAlignments:
+  chains: ALL
+  noHeader: false
+  fields:
+  - field: -readIds
+  - field: -targetSequences
+  - field: -targetQualities
+  - field: -vHitsWithScore
+  - field: -dHitsWithScore
+  - field: -jHitsWithScore
+  - field: -cHitsWithScore
+  - field: -vAlignments
+  - field: -dAlignments
+  - field: -jAlignments
+  - field: -cAlignments
+  - field: -allNFeaturesWithMinQuality
+  - field: -allAAFeatures
+  - field: -defaultAnchorPoints
 exportClones:
   splitByTagType: null
   filterOutOfFrames: false

diff --git a/regression/presets/analyze/bd-rhapsody-mouse-bcr-full-length.yaml b/regression/presets/analyze/bd-rhapsody-mouse-bcr-full-length.yaml
@@ -388,6 +388,7 @@ exportAlignments:
   chains: ALL
   noHeader: false
   fields:
+  - field: -readIds
   - field: -targetSequences
   - field: -targetQualities
   - field: -vHitsWithScore

diff --git a/regression/presets/analyze/bd-rhapsody-mouse-bcr-v1.yaml b/regression/presets/analyze/bd-rhapsody-mouse-bcr-v1.yaml
@@ -424,7 +424,24 @@ assemble:
       requiredSequences: []
   inferMinRecordsPerConsensus: true
 assembleContigs: null
-exportAlignments: null
+exportAlignments:
+  chains: ALL
+  noHeader: false
+  fields:
+  - field: -readIds
+  - field: -targetSequences
+  - field: -targetQualities
+  - field: -vHitsWithScore
+  - field: -dHitsWithScore
+  - field: -jHitsWithScore
+  - field: -cHitsWithScore
+  - field: -vAlignments
+  - field: -dAlignments
+  - field: -jAlignments
+  - field: -cAlignments
+  - field: -allNFeaturesWithMinQuality
+  - field: -allAAFeatures
+  - field: -defaultAnchorPoints
 exportClones:
   splitByTagType: null
   filterOutOfFrames: false

diff --git a/regression/presets/analyze/bd-rhapsody-mouse-bcr-v2.yaml b/regression/presets/analyze/bd-rhapsody-mouse-bcr-v2.yaml
@@ -427,7 +427,24 @@ assemble:
       requiredSequences: []
   inferMinRecordsPerConsensus: true
 assembleContigs: null
-exportAlignments: null
+exportAlignments:
+  chains: ALL
+  noHeader: false
+  fields:
+  - field: -readIds
+  - field: -targetSequences
+  - field: -targetQualities
+  - field: -vHitsWithScore
+  - field: -dHitsWithScore
+  - field: -jHitsWithScore
+  - field: -cHitsWithScore
+  - field: -vAlignments
+  - field: -dAlignments
+  - field: -jAlignments
+  - field: -cAlignments
+  - field: -allNFeaturesWithMinQuality
+  - field: -allAAFeatures
+  - field: -defaultAnchorPoints
 exportClones:
   splitByTagType: null
   filterOutOfFrames: false