diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 00000000000..6ee8284567e --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# +# Reformats staged .java files with IntelliJ's command-line formatter using the +# repo's hmf_code_scheme.xml, so commits match the in-IDE style. +# +# One-time setup per clone: git config core.hooksPath .githooks +# Skip a single commit: git commit --no-verify +# Override the formatter: export FORMAT_SH=/path/to/format.sh +# +set -euo pipefail + +repo_root="$(git rev-parse --show-toplevel)" +scheme="$repo_root/hmf_code_scheme.xml" + +# Staged Java files (added/copied/modified), NUL-safe (bash 3.2 compatible). +java_files=() +while IFS= read -r -d '' f; do + java_files+=("$f") +done < <(git diff --cached --name-only --diff-filter=ACM -z -- '*.java') +[[ ${#java_files[@]} -eq 0 ]] && exit 0 + +# Resolve IntelliJ's format.sh. +format_sh="${FORMAT_SH:-}" +if [[ -z "$format_sh" ]]; then + for candidate in \ + "/Applications/IntelliJ IDEA.app/Contents/bin/format.sh" \ + "/Applications/IntelliJ IDEA CE.app/Contents/bin/format.sh" \ + "$HOME/.local/share/JetBrains/Toolbox/apps/"*/*/bin/format.sh \ + /opt/intellij*/bin/format.sh; do + [[ -x "$candidate" ]] && { format_sh="$candidate"; break; } + done +fi + +if [[ -z "$format_sh" || ! -x "$format_sh" ]]; then + echo "pre-commit: IntelliJ format.sh not found; skipping formatting." >&2 + echo " Set FORMAT_SH=/path/to/format.sh to enable." >&2 + exit 0 +fi + +# Absolute paths for the formatter. +abs_files=() +for f in "${java_files[@]}"; do + abs_files+=("$repo_root/$f") +done + +# Run with an isolated config/system dir so the formatter doesn't collide with +# an already-open IntelliJ ("Only one instance of IDEA can be run at a time"). +# Cached under .git for warm-start speed. +hook_idea_dir="$(git rev-parse --git-path hmf-format-idea)" +mkdir -p "$hook_idea_dir" +hook_idea_dir="$(cd "$hook_idea_dir" && pwd)" +props="$hook_idea_dir/idea.properties" +printf 'idea.config.path=%s/config\nidea.system.path=%s/system\n' "$hook_idea_dir" "$hook_idea_dir" > "$props" +export IDEA_PROPERTIES="$props" + +echo "pre-commit: formatting ${#abs_files[@]} Java file(s) with IntelliJ scheme..." +if ! "$format_sh" -s "$scheme" "${abs_files[@]}"; then + echo "pre-commit: format.sh failed; commit with --no-verify to bypass." >&2 + exit 1 +fi + +# Re-stage anything the formatter touched. +for f in "${java_files[@]}"; do + git add -- "$repo_root/$f" +done diff --git a/entrypoint_template.sh b/entrypoint_template.sh deleted file mode 100644 index d7d4adb13b0..00000000000 --- a/entrypoint_template.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -set -euo pipefail - -# Disable glob expansion. "*" in filenames are handled within hmftools java code -set -f - -# Parse arguments -jvm_mem_opts="" -jvm_gen_opts="" -declare -a app_args -for arg in "$@"; do - case ${arg} in - '-Xm'*) - jvm_mem_opts="${jvm_mem_opts} ${arg}" - ;; - '-D'* | '-XX'*) - jvm_gen_opts="${jvm_gen_opts} ${arg}" - ;; - *) - app_args+=("${arg}") - ;; - esac -done - -# Form and execute command -if [[ ${app_args[0]:=} == com.hartwig.* ]]; then - java ${jvm_mem_opts} ${jvm_gen_opts} -cp __JAR_PATH__ ${app_args[@]} -else - java ${jvm_mem_opts} ${jvm_gen_opts} -jar __JAR_PATH__ ${app_args[@]} -fi - diff --git a/hmf-common/src/main/java/com/hartwig/hmftools/common/bam/CigarUtils.java b/hmf-common/src/main/java/com/hartwig/hmftools/common/bam/CigarUtils.java index 70ed91a9fd9..1a7cc9bd582 100644 --- a/hmf-common/src/main/java/com/hartwig/hmftools/common/bam/CigarUtils.java +++ b/hmf-common/src/main/java/com/hartwig/hmftools/common/bam/CigarUtils.java @@ -7,10 +7,12 @@ import static com.hartwig.hmftools.common.bam.SamRecordUtils.NO_CIGAR; import static com.hartwig.hmftools.common.bam.SamRecordUtils.NO_POSITION; +import static htsjdk.samtools.CigarOperator.EQ; import static htsjdk.samtools.CigarOperator.H; import static htsjdk.samtools.CigarOperator.I; import static htsjdk.samtools.CigarOperator.M; import static htsjdk.samtools.CigarOperator.S; +import static htsjdk.samtools.CigarOperator.X; import java.util.ArrayList; import java.util.Collections; @@ -77,6 +79,34 @@ public static int cigarAlignedLength(final Cigar cigar) return cigar.getCigarElements().stream().filter(x -> x.getOperator().consumesReferenceBases()).mapToInt(x -> x.getLength()).sum(); } + public static int cigarBaseLength(final List elements) + { + return elements.stream().filter(x -> x.getOperator().consumesReadBases()).mapToInt(CigarElement::getLength).sum(); + } + + public static int cigarAlignedLength(final List elements) + { + return elements.stream().filter(x -> x.getOperator().consumesReferenceBases()).mapToInt(CigarElement::getLength).sum(); + } + + // total M/=/X bases (aligned bases excluding indels and skips); matches STAR's overhang definition + public static int matchedBases(final List elements) + { + int total = 0; + for(CigarElement element : elements) + { + CigarOperator operator = element.getOperator(); + if(operator == M || operator == EQ || operator == X) + total += element.getLength(); + } + return total; + } + + public static boolean hasHardClip(final List elements) + { + return elements.stream().anyMatch(x -> x.getOperator() == H); + } + public static int calcCigarAlignedLength(final String cigarStr) { // a string-parsing version of the method above @@ -142,6 +172,12 @@ public static int rightSoftClipLength(final Cigar cigar) return (lastElement != null && lastElement.getOperator() == S) ? lastElement.getLength() : 0; } + public static int rightSoftClipLength(final List elements) + { + CigarElement lastElement = elements.get(elements.size() - 1); + return lastElement.getOperator() == S ? lastElement.getLength() : 0; + } + public static int leftHardClipLength(final Cigar cigar) { CigarElement firstElement = cigar.getFirstCigarElement(); diff --git a/hmf-common/src/main/java/com/hartwig/hmftools/common/bam/SamRecordUtils.java b/hmf-common/src/main/java/com/hartwig/hmftools/common/bam/SamRecordUtils.java index 941e3e7839b..f4acd83509d 100644 --- a/hmf-common/src/main/java/com/hartwig/hmftools/common/bam/SamRecordUtils.java +++ b/hmf-common/src/main/java/com/hartwig/hmftools/common/bam/SamRecordUtils.java @@ -40,6 +40,7 @@ public final class SamRecordUtils public static final String READ_GROUP_ATTRIBUTE = SAMTag.RG.name(); public static final String XS_ATTRIBUTE = "XS"; + public static final String XA_ATTRIBUTE = "XA"; // Redux tags public static final String CONSENSUS_READ_ATTRIBUTE = "CR"; diff --git a/hmf-common/src/main/java/com/hartwig/hmftools/common/logging/HmfLogger.java b/hmf-common/src/main/java/com/hartwig/hmftools/common/logging/HmfLogger.java new file mode 100644 index 00000000000..1f723b68366 --- /dev/null +++ b/hmf-common/src/main/java/com/hartwig/hmftools/common/logging/HmfLogger.java @@ -0,0 +1,43 @@ +package com.hartwig.hmftools.common.logging; + +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.spi.ExtendedLogger; +import org.apache.logging.log4j.spi.ExtendedLoggerWrapper; + +// Logger with a custom DEBUG_2 level: finer than DEBUG (intLevel 500), coarser than TRACE (600), so it can be enabled to show +// extra per-read detail without pulling in third-party TRACE noise (htsjdk and friends log at TRACE). The wrapper mirrors what +// log4j2's own custom-level generator produces, exposing TARS_LOGGER.debug2(...) alongside the standard level methods. +public final class HmfLogger extends ExtendedLoggerWrapper +{ + public static final Level DEBUG_2 = Level.forName("DEBUG_2", 550); + + private static final String FQCN = HmfLogger.class.getName(); + + private HmfLogger(final ExtendedLogger logger) + { + super(logger, logger.getName(), logger.getMessageFactory()); + } + + public static HmfLogger getLogger(final Class clazz) + { + return new HmfLogger((ExtendedLogger) LogManager.getLogger(clazz)); + } + + public static HmfLogger getLogger(final String name) + { + return new HmfLogger((ExtendedLogger) LogManager.getLogger(name)); + } + + // Resolves standard and hmftools custom level names. Referencing this class loads DEBUG_2 (via the static field above) so a + // later Level.valueOf("DEBUG_2") does not throw. + public static Level parseLevel(final String name) + { + return Level.valueOf(name); + } + + public void debug2(final String message, final Object... params) + { + logIfEnabled(FQCN, DEBUG_2, null, message, params); + } +} diff --git a/hmf-common/src/main/java/com/hartwig/hmftools/common/utils/config/ConfigUtils.java b/hmf-common/src/main/java/com/hartwig/hmftools/common/utils/config/ConfigUtils.java index 1e55ecfdc9b..95701166b75 100644 --- a/hmf-common/src/main/java/com/hartwig/hmftools/common/utils/config/ConfigUtils.java +++ b/hmf-common/src/main/java/com/hartwig/hmftools/common/utils/config/ConfigUtils.java @@ -5,6 +5,8 @@ import static com.hartwig.hmftools.common.utils.file.FileDelimiters.CSV_DELIM; import static com.hartwig.hmftools.common.utils.file.FileReaderUtils.createFieldsIndexMap; +import com.hartwig.hmftools.common.logging.HmfLogger; + import java.io.File; import java.io.IOException; import java.nio.file.Files; @@ -26,7 +28,7 @@ public class ConfigUtils public static final String LOG_DEBUG_DESC = "Log at DEBUG level"; public static final String LOG_LEVEL = "log_level"; - public static final String LOG_LEVEL_DESC = "Specify log level: ERROR, WARN, INFO, DEBUG or TRACE"; + public static final String LOG_LEVEL_DESC = "Specify log level: ERROR, WARN, INFO, DEBUG, DEBUG_2 or TRACE"; public static final String SAMPLE_ID_FILE = "sample_id_file"; public static final String SAMPLE_ID_FILE_DESC = "Sample ID CSV file with 'SampleId' column"; @@ -54,7 +56,7 @@ public static void setLogLevel(final ConfigBuilder configBuilder) } else if(configBuilder.isRegistered(LOG_LEVEL) && configBuilder.hasValue(LOG_LEVEL)) { - Configurator.setRootLevel(Level.valueOf(configBuilder.getValue(LOG_LEVEL))); + Configurator.setRootLevel(HmfLogger.parseLevel(configBuilder.getValue(LOG_LEVEL))); } } diff --git a/hmf-common/src/main/java/com/hartwig/hmftools/common/utils/config/LoggingOptions.java b/hmf-common/src/main/java/com/hartwig/hmftools/common/utils/config/LoggingOptions.java index ce2096eb0e2..95fe5cd49b8 100644 --- a/hmf-common/src/main/java/com/hartwig/hmftools/common/utils/config/LoggingOptions.java +++ b/hmf-common/src/main/java/com/hartwig/hmftools/common/utils/config/LoggingOptions.java @@ -2,6 +2,8 @@ import org.apache.logging.log4j.core.config.Configurator; import org.apache.logging.log4j.Level; +import com.hartwig.hmftools.common.logging.HmfLogger; + import com.beust.jcommander.IStringConverter; import com.beust.jcommander.Parameter; @@ -19,7 +21,7 @@ static class Log4jLevelTypeConverter implements IStringConverter @Override public Level convert(String value) { - return Level.valueOf(value); + return HmfLogger.parseLevel(value); } } @@ -27,7 +29,7 @@ public Level convert(String value) public boolean LogDebug; @Parameter(names = "-" + ConfigUtils.LOG_LEVEL, - description = "Specify log level (values: [OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE, ALL])", + description = "Specify log level (values: [OFF, FATAL, ERROR, WARN, INFO, DEBUG, DEBUG_2, TRACE, ALL])", converter = Log4jLevelTypeConverter.class) public org.apache.logging.log4j.Level LogLevel = null; diff --git a/hmf-common/src/test/java/com/hartwig/hmftools/common/bam/CigarUtilsTest.java b/hmf-common/src/test/java/com/hartwig/hmftools/common/bam/CigarUtilsTest.java index 1271ffdbefa..47cd348a582 100644 --- a/hmf-common/src/test/java/com/hartwig/hmftools/common/bam/CigarUtilsTest.java +++ b/hmf-common/src/test/java/com/hartwig/hmftools/common/bam/CigarUtilsTest.java @@ -33,6 +33,20 @@ public class CigarUtilsTest { + @Test + public void testElementListHelpers() + { + final List elements = CigarUtils.cigarElementsFromStr("10S94M79N57M3S"); + + assertEquals(164, CigarUtils.cigarBaseLength(elements)); // read-consuming: 10S + 94M + 57M + 3S + assertEquals(230, CigarUtils.cigarAlignedLength(elements)); // ref-consuming: 94M + 79N + 57M + assertEquals(151, CigarUtils.matchedBases(elements)); // M/=/X only: 94M + 57M + assertEquals(10, CigarUtils.leftSoftClipLength(elements)); + assertEquals(3, CigarUtils.rightSoftClipLength(elements)); + assertFalse(CigarUtils.hasHardClip(elements)); + assertTrue(CigarUtils.hasHardClip(CigarUtils.cigarElementsFromStr("5H94M"))); + } + @Test public void testCigarFromStr() { diff --git a/hmf-common/src/test/java/com/hartwig/hmftools/common/logging/HmfLoggerTest.java b/hmf-common/src/test/java/com/hartwig/hmftools/common/logging/HmfLoggerTest.java new file mode 100644 index 00000000000..74e92f3c2f2 --- /dev/null +++ b/hmf-common/src/test/java/com/hartwig/hmftools/common/logging/HmfLoggerTest.java @@ -0,0 +1,40 @@ +package com.hartwig.hmftools.common.logging; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.apache.logging.log4j.Level; +import org.junit.Test; + +public class HmfLoggerTest +{ + @Test + public void debug2RegisteredAndParseable() + { + assertEquals(HmfLogger.DEBUG_2, HmfLogger.parseLevel("DEBUG_2")); + assertEquals(550, HmfLogger.DEBUG_2.intLevel()); + } + + @Test + public void parseLevelResolvesStandardLevels() + { + assertEquals(Level.INFO, HmfLogger.parseLevel("INFO")); + assertEquals(Level.TRACE, HmfLogger.parseLevel("TRACE")); + } + + @Test + public void debug2SitsBetweenDebugAndTrace() + { + // finer than DEBUG (shows everything DEBUG does), but coarser than TRACE (excludes third-party TRACE noise) + assertTrue(HmfLogger.DEBUG_2.isLessSpecificThan(Level.DEBUG)); + assertTrue(HmfLogger.DEBUG_2.isMoreSpecificThan(Level.TRACE)); + } + + @Test + public void debug2MethodLogsWithoutError() + { + // smoke: the extended-logger method resolves and runs with parameterized placeholders + HmfLogger logger = HmfLogger.getLogger(HmfLoggerTest.class); + logger.debug2("debug2 smoke {} {}", "a", 1); + } +} diff --git a/pom.xml b/pom.xml index 4a4dca69c98..afa278b5be3 100644 --- a/pom.xml +++ b/pom.xml @@ -42,6 +42,7 @@ sage sigs sv-tools + tars teal virus-interpreter v-chord @@ -95,6 +96,7 @@ 5.1 1.2.1 1.0 + 1.0 1.4 1.7.1 1.0.1 diff --git a/redux/src/main/java/com/hartwig/hmftools/redux/duplicate/ReadCache.java b/redux/src/main/java/com/hartwig/hmftools/redux/duplicate/ReadCache.java index f258fa97f54..a1ba70054af 100644 --- a/redux/src/main/java/com/hartwig/hmftools/redux/duplicate/ReadCache.java +++ b/redux/src/main/java/com/hartwig/hmftools/redux/duplicate/ReadCache.java @@ -55,6 +55,8 @@ public class ReadCache public static final int DEFAULT_POP_DISTANCE_CHECK = 100; // how often in base terms to check for popping read groups public static final int DEFAULT_LOG_READ_COUNT_THRESHOLD = 100000; // based on observed cache sizes for deep panels public static final int DEFAULT_DYNAMIC_READ_COUNT_THRESHOLD = 100_000; // level at which steps are taken to reduce the cache size + // log a single fragment-coord group reaching this many reads (extreme-expression locus) + public static final int LARGE_FRAG_GROUP_LOG_SIZE = 1000; public ReadCache( int groupSize, int maxSoftClipLength, boolean useFragmentOrientation, final DuplicatesConfig duplicatesConfig, @@ -465,6 +467,12 @@ public void addRead(final FragmentCoords fragCoords, final SAMRecord read) } reads.add(read); + + if(reads.size() == LARGE_FRAG_GROUP_LOG_SIZE) + { + RD_LOGGER.debug("large frag-coord group({} reads) at {}:{} cigar({})", + reads.size(), Chromosome, read.getAlignmentStart(), read.getCigarString()); + } } public int readCount() { return FragCoordsMap.values().stream().mapToInt(x -> x.size()).sum(); } diff --git a/redux/src/main/java/com/hartwig/hmftools/redux/write/BamWriterNoSync.java b/redux/src/main/java/com/hartwig/hmftools/redux/write/BamWriterNoSync.java index 7afe4601868..14cfc1bb974 100644 --- a/redux/src/main/java/com/hartwig/hmftools/redux/write/BamWriterNoSync.java +++ b/redux/src/main/java/com/hartwig/hmftools/redux/write/BamWriterNoSync.java @@ -105,9 +105,10 @@ public void close() { mSortedBamWriter.flush(); - RD_LOGGER.debug("sorted-writer records written({} writes={} avg={} max={}) maxCache({}) to BAM({})", + RD_LOGGER.debug("sorted-writer records written({} writes={} avg={} max={}) maxCache({} at {}) to BAM({})", mSortedBamWriter.written(), mSortedBamWriter.writeCount(), mSortedBamWriter.avgWriteCount(), - mSortedBamWriter.maxWrite(), mSortedBamWriter.maxCache(), filenamePart(mFilename)); + mSortedBamWriter.maxWrite(), mSortedBamWriter.maxCache(), mSortedBamWriter.maxCacheLocation(), + filenamePart(mFilename)); } else if(mBamWriter != null) { diff --git a/redux/src/main/java/com/hartwig/hmftools/redux/write/SortedBamWriter.java b/redux/src/main/java/com/hartwig/hmftools/redux/write/SortedBamWriter.java index 87667f209ff..1e4fdc5e6f8 100644 --- a/redux/src/main/java/com/hartwig/hmftools/redux/write/SortedBamWriter.java +++ b/redux/src/main/java/com/hartwig/hmftools/redux/write/SortedBamWriter.java @@ -39,6 +39,9 @@ public class SortedBamWriter private long mReadCount; private long mReadsWritten; private int mMaxCached; + private String mMaxCachedChromosome; + private int mMaxCachedPosition; + private String mMaxCachedCigar; private int mMaxWrite; private boolean mPerfDebug; @@ -111,6 +114,9 @@ public SortedBamWriter(final SortedBamConfig config, @Nullable final SAMFileWrit mWriteCount = 0; mReadCount = 0; mMaxCached = 0; + mMaxCachedChromosome = ""; + mMaxCachedPosition = 0; + mMaxCachedCigar = ""; mMaxWrite = 0; mPerfDebug = false; } @@ -151,7 +157,14 @@ public boolean canWriteRecord(final SAMRecord read) public void addRecord(final SAMRecord read) { mRecords.add(new ReadOrAlignmentStart(mReadCount++, read)); - mMaxCached = max(mMaxCached, mRecords.size()); + + if(mRecords.size() > mMaxCached) + { + mMaxCached = mRecords.size(); + mMaxCachedChromosome = read.getReferenceName(); + mMaxCachedPosition = read.getAlignmentStart(); + mMaxCachedCigar = read.getCigarString(); + } if(mMinCachedPosition == 0) mMinCachedPosition = read.getAlignmentStart(); @@ -215,6 +228,7 @@ public void flush() public long writeCount() { return mWriteCount; } public int cached() { return mRecords.size(); } public int maxCache() { return mMaxCached; } + public String maxCacheLocation() { return format("%s:%d cigar(%s)", mMaxCachedChromosome, mMaxCachedPosition, mMaxCachedCigar); } public int maxWrite() { return mMaxWrite; } public int avgWriteCount() { return mWriteCount > 0 ? (int)(mReadsWritten / (double)mWriteCount) : 0; } diff --git a/tars/README.md b/tars/README.md new file mode 100644 index 00000000000..5f62600a0df --- /dev/null +++ b/tars/README.md @@ -0,0 +1,213 @@ +# TARS + +**TARS** (Transcript Alignment for RNA Splicing) makes `bwa-mem2` splice-aware for RNA reads. TARS aligns the reads against the genome with the +transcriptome added using `bwa-mem2`, then rewrites the result back to genome coordinates. The output is an ordinary genomic RNA +BAM (no transcript contigs, spliced reads carried as `N` gaps) ready for REDUX and ISOFOX. + +## Contents + +* [What TARS does](#what-tars-does) +* [How to Run TARS](#how-to-run-tars) +* [What a read goes through](#what-a-read-goes-through) + * [Step 0: Translate transcriptome alignments to reference genome](#step-0-translate-transcriptome-alignments-to-reference-genome) + * [Step 1: Score short overhangs against the reference genome, collapse weak scoring ones](#step-1-score-short-overhangs-against-the-reference-genome-collapse-weak-scoring-ones) + * [Step 2: Decide which alignments to keep for a read](#step-2-decide-which-alignments-to-keep-for-a-read) + * [Step 3: Resolve supplementary records into splice junctions](#step-3-resolve-supplementary-records-into-splice-junctions) + +## What TARS does + +A normal genome aligner like `bwa-mem2` matches a read against one continuous stretch of genome. It has no idea where +introns are, so a read that jumps over one gets cut short or placed in the wrong spot. + +TARS fixes this without changing the aligner: + +1. **`SpliceFastaBuilder`** (run once per Ensembl release) concatenates each multi-exon transcript's exon sequences into + a transcript contig (`*_tx`, introns removed); these contigs are the transcriptome. It also writes a sidecar TSV + mapping each contig's intervals back to their genomic exon spans. +2. Append the transcriptome to the genome FASTA and index with `bwa-mem2`. +3. Align the RNA reads with `bwa-mem2` as usual. A read that jumps an intron now has a continuous place to land. +4. **`TarsApplication`** lifts each read back to its real genome position, marks the skipped intron as a gap (`N`), fixes + things up (tags, mate info, confidence), and writes a sorted, indexed BAM. +5. Feed the new splice-aware records BAM file into REDUX (dedup), then ISOFOX. + +![TARS pipeline](doc/tars.png) + +## How to Run TARS + +### Build the transcript reference (SpliceFastaBuilder) + +``` +java -cp tars.jar com.hartwig.hmftools.tars.fasta.SpliceFastaBuilder + -ensembl_data_dir /ref_data/ensembl_data_cache/38/ + -ref_genome /path_to_fasta/genome.fasta + -ref_genome_version V38 + -output_dir /path_to_output/ +``` + +Two files are written: + +* `ref_genome_v38_rna_contigs.fasta` - the transcript contigs. +* `ref_genome_v38_rna_contigs.rna_contigs_mappings.tsv` - the contig sidecar (intervals mapped back to genomic exon spans). + +Concatenate the FASTA onto the genome FASTA and `bwa-mem2 index` the result before aligning. + +### Run TARS (TarsApplication) + +``` +java -jar tars.jar + -sample ACTN01020030T + -input_bam ACTN01020030T.bwa_tx.namegrouped.bam + -ref_genome /path_to_fasta/genome_plus_tx.fasta + -ref_genome_version V38 + -contig_sidecar /path_to/ref_genome_v38_rna_contigs.rna_contigs_mappings.tsv + -rna_unmap_regions /ref_data/rna/38/rna_excluded_regions.38.tsv + -bamtool /path_to_samtools/ + -output_dir /path_to_output/ + -threads 24 +``` + +### Output files + +Every file is named `.tars.<...>`. Two are written by default: + +* `.tars.bam` (+ `.bai`) - the lifted, coord-sorted genomic BAM, ready for REDUX. +* `.tars.summary.tsv` - a counts summary of what liftback did. + +Optional: + +* `-output_id chr1_slice` inserts the token into every name: `.tars.chr1_slice.bam`. +* `-write_liftback_tsv` writes per-record debug TSVs; off by default (~100GB; per-read detail the summary can't give). + +### Flags + +**Required** + +| Flag | Description | +|--------------------|------------------------------------------------------------------------------| +| sample | Sample ID. prefix to each output file (`.tars.*`) | +| input_bam | bwa-mem2 output against the combined FASTA, **name-grouped** (not coord-sorted)| +| ref_genome | The same combined genome + transcript FASTA used at alignment | +| ref_genome_version | `V37` or `V38` | +| contig_sidecar | Contig sidecar TSV from `SpliceFastaBuilder` (`*.rna_contigs_mappings.tsv`) | +| bamtool | samtools path (used to decompress the input and sort + index output) | +| output_dir | Directory for the lifted BAM and summary file | + +**Optional** + +| Flag | Default | Description | +|--------------------|---------|--------------------------------------------------------------------------| +| output_id | (none) | id inserted into every output files | +| rna_unmap_regions | (none) | Curated excluded regions (rRNA / 7SL / multi-map zones) whose reads are removed from the lifted output; see [rna_excluded_regions.38.tsv](https://source.cloud.google.com/hmf-pipeline-development/common-resources-public/+/master:rna/38/rna_excluded_regions.38.tsv) | +| write_liftback_tsv | off | Per-record debug TSVs; off by default (creates a `~100GB` file) | +| threads | 1 | Worker threads; reads process in parallel per read-group | +| log_level | INFO | `DEBUG` & `DEBUG2` are other levels | + +**Tuning thresholds** + +| Flag | Default | Description | +|---------------------------------|-----------|---------------------------------------------------------------| +| supp_implied_min_intron_length | 21 | Min implied intron length for a primary+supp merge | +| supp_implied_max_intron_length | 1000000 | Max implied intron length for a primary+supp merge | + +Note: no `ensembl_data_dir` - liftback reads exon/junction annotation from the sidecar (only `SpliceFastaBuilder` needs +ensembl). + +### Upstream bwa-mem2 flags + +Not tars config, but liftback depends on them. + +| Setting | Default | What it is | +|---|---|---| +| `-T` | 19 | bwa-mem2 minimum alignment score to output; set below the default 30 to surface short-anchor supplementaries for supplementary resolve | +| `-h` | 75 | bwa-mem2 XA cap; maximum alternate loci listed per read before it is unmapped | + +## What a read goes through + +After bwa-mem2 alignment, a read that spans an exon boundary or has supplementaries around novel junctions is processed by +tars through these steps in order: + +``` +Step 0 Translate lift every candidate (primary + XA alts) to genome coordinates +Step 1 Overhang re-evaluate each overhang and collapse the weak scoring ones +Step 2 Decide choose which alignments to keep as the primary and its XA alternates +Step 3 Merge resolve supplementary records into splice junctions +``` + +### Step 0: Translate transcriptome alignments to reference genome + +Every read's transcriptome alignment is translated to genomic coordinates, with introns re-inserted as `N` gaps / splice junctions. + +![translate the read to the genome](doc/translate.svg) + +### Step 1: Score short overhangs against the reference genome, collapse weak scoring ones + +A short overhang (`<= 12M`) next to a splice junction at a read end is re-scored using bwa-mem2 style scoring against the +reference genome. There are 3 cases: + +**1a.** With a soft clip: keep the junction if the overhang scores > 5; otherwise drop the `N` junction and walk the soft +clip onto the reference genome, leaving a contiguous alignment. + +![1 splice junction](doc/overhang_one_junction.svg) + +**1b.** With >1 splice junctions: keep the junction if the short overhang aligns positively (AS > 0), otherwise collapse +it only when the intronic reference AS > short overhang AS. + +![more than 1 splice junction](doc/overhang_two_junctions.svg) + +**1c.** With no soft clip and a single junction: not checked, no intervention. + +### Step 2: Decide which alignments to keep for a read + +A read arrives with one or more candidate placements - its own alignment plus any `XA` alternates - each a genomic (ref) +or transcriptome (tx) alignment. TARS picks one as the primary, keeps the rest in `XA`, and sets its `MAPQ`. Every read +lands in one of four buckets: + +- **Ref only** - kept as bwa placed it. A `MAPQ 0` read with no `XA` has exceeded bwa's `-h` cap (too many loci to + report) and is unmapped instead. + +- **Single transcriptome locus** - one or more transcript contigs of the same gene lift to a single genomic locus with + the same CIGAR. TARS places the read back with `N` gaps and, as a unique placement, raises `MAPQ` to 60. + +- **Multi-mapper** - several loci with no single best. Every placement is kept in `XA` and `MAPQ` stays below 60 (`0` + when fully tied). The primary is the highest-scoring locus (scoring below); if bwa already ranked the loci its order + stands, and a genuine score tie is settled by a read-name seed so tied reads spread across their loci instead of + piling onto one. A single-locus read with `XS == AS` hides an equal-scoring placement bwa did not report, so it stays + at `MAPQ 0` unless a transcript match or annotated exon confirms it. + +- **Contested ref vs tx** - a tx alignment (spliced, with an `N`) and a ref alignment disagree. The tx junction wins + unless the ref reads cleanly through it: + - the ref is contiguous at another locus, or soft-clips at the boundary - it cannot cross the junction, so keep the + spliced placement (swap the primary to it, or drop the ref alt) + - the ref is a full match straight through the gap - the read did not splice here (unspliced pre-mRNA, retained + intron, or DNA contamination), so keep the ref placement and drop the `N` + - neither fires and bwa gave no priority (`MAPQ 0`) - the higher-scoring side wins (below), else a read-name-seeded + pick; the loser stays in `XA`, `MAPQ` at 0 + +Both seeded picks above are decided first by a recomputed genome-space `bwa-mem` score, and use the read-name seed only +on an exact tie (equal recomputed scores); reads with a supplementary are left for Step 3. + +### Step 3: Resolve supplementary records into splice junctions + +`bwa-mem2` is run with `-T 19`, allowing short-anchor supplementary alignments at junction sites (annotated or novel) to +be kept. Given the pre-requisites, TARS merges such a supplementary back into its primary with a splice junction and drops +the supplementary. The merge requires: + +- the primary and supplementary are within reach and complementary to each other's soft clips +- the implied intron length is within [`supp_implied_min_intron_length`, `supp_implied_max_intron_length`] +- exactly one supplementary is within reach on that side, otherwise it is ambiguous and left unmerged + +On a successful resolve, it's still ambiguous where the splice junction is. TARS attempts in this order: + +1. an annotated boundary / known junction (Ensembl) +2. a canonical `GT-AG` splice motif, then semi-canonical +3. the mate's already-resolved junction +4. the midpoint of the ambiguous read range, rounded down +5. otherwise keep whatever was chosen + +When several positions tie at the chosen tier, the pick is pseudo-random but seeded by the read, so an ambiguous +junction is distributed across its equal options yet stays reproducible run to run. + +The merged primary's MAPQ is `max(primary, supplementary)`, bumped to 60 when the primary + supplementary pair maps to a +single locus (no competing alternative alignment). + +![merge supplementary record to primary](doc/rescue_via_supplementary.svg) \ No newline at end of file diff --git a/tars/doc/_template.svg b/tars/doc/_template.svg new file mode 100644 index 00000000000..293cec9869b --- /dev/null +++ b/tars/doc/_template.svg @@ -0,0 +1,87 @@ + + + + + + + + + + + + + + + + + + + + + + matched (M) + + exon (genome) + + softclip (S) + + intron (N) + + kept + + + +replace me with the diagram + diff --git a/tars/doc/core_splice_tx_only.svg b/tars/doc/core_splice_tx_only.svg new file mode 100644 index 00000000000..e2660e6c2d9 --- /dev/null +++ b/tars/doc/core_splice_tx_only.svg @@ -0,0 +1,109 @@ + +Core splice (tx-only), MAPQ 0 to 60 +A read aligns to the transcriptome as 151M (three packed copies), all tied at MAPQ 0. Translation lifts it across a 7,762 bp intron to a single spliced genomic locus chr22:45,923,222 as 114M 7762N 37M; the redundant copies collapse to one locus and MAPQ is rescued from 0 to 60 (NH 1). + + + + + + + + + + + + + + + + + + + +matched (M) + +exon (genome) + +intron (N) + + + +before · read aligns to the transcriptome (3 packed copies, all tied) + +151M · chr22_tx:8,725,640 + +151M · copy 2 (XA, NM 0) + +151M · copy 3 (XA, NM 0) + +MAPQ 0 +no single best hit (all copies score equally) so bwa emits MAPQ 0 + + + +translate across the intron · copies collapse to one locus + + +reference genome · chr22 + + + +exon +exon + +45,923,222 +7,762 bp (not to scale) + + +after · one spliced genomic placement + +114M + + +7762N + +37M + +MAPQ 60 +MAPQ bumped to 60 = liftback asserting a rescue (single best genomic locus) + + diff --git a/tars/doc/junction_tx_spans_ref_clips.svg b/tars/doc/junction_tx_spans_ref_clips.svg new file mode 100644 index 00000000000..24d6ebe1022 --- /dev/null +++ b/tars/doc/junction_tx_spans_ref_clips.svg @@ -0,0 +1,129 @@ + +JUNCTION: tx spans the intron, ref softclips at the boundary +At one locus, chr1:1,049,917, the bwa primary is a transcript alignment that spans a 195 bp intron (121M 195N 30M); the ref alt aligns 115M then softclips 36 bases because it cannot read across the intron. The 30 clipped bases match the next exon, not the intron, so tars keeps the spliced tx placement, drops the ref alt, and rescues MAPQ 0 to 60. + + + + + + + + + + + + + + + + + + + +matched (M) + +softclip (S) + +exon (genome) + +intron (N) + + + +before · two placements at one locus, chr1:1,049,917 + + +bwa primary (tx): spans the 195 bp intron onto the next exon + +121M + + +195N + +30M + +MAPQ 0 + + +ref alt (XA): aligns 115M then softclips - cannot cross the intron + +115M + +36S + + +reference genome · chr1 + + + +exon +next exon + +1,049,917 +195 bp (not to scale) + +exon end · ref clips, tx splices + + + +the 30 clipped bases match the next exon, not the intron (the 195 bp intron opens GT): +read[122..151] +CCTCGGGGCAGGACGGCTCTGGGCCCTTCG +chr1:1,050,233 +CCTCGGGGCAGGACGGCTCTGGGCCCTTCC +29/30 + + +after · tx kept, ref alt dropped + +121M + + +195N + +30M + +MAPQ 60 +tx explains the whole read across a real junction; no XA written on a drop (NH 1) + +tx wins · the read crosses a junction the ref can only softclip + diff --git a/tars/doc/overhang_one_junction.svg b/tars/doc/overhang_one_junction.svg new file mode 100644 index 00000000000..1eb03c79463 --- /dev/null +++ b/tars/doc/overhang_one_junction.svg @@ -0,0 +1,75 @@ + +Step 1: one splice junction, kept or collapsed +Short overhangs (12 bp or fewer) next to a soft clip, decided by the overhang score (bar = 5). Bar widths are proportional to base count; introns (N) are drawn as fixed gaps. Kept: chr5:181,241,515, 125M 534N 11M 15S, the 11M overhang matches its far exon (AS +11) so the junction is kept. Collapsed: the exp8 field read 25535, 100M 83N 3M 48S, has a 3 bp anchor scoring 3, so the junction is dropped to 100M 51S. Collapsed by SNV: a 7 bp anchor with one mismatch scores 6 minus 4 equals 2, below the bar, so a genuine junction is collapsed to 100M 51S even though its CIGAR M is longer. Score counts matches, not M length. M + S totals 151. + + + + + + + + + +1 splice junction + +BEFORE +AFTER + + + + +kept · 11 bp overhang matches its far exon (AS +11) +125M +534N +11M +15S +125M +534N +11M +15S + + + +collapsed · exp8 read 25535, 3 bp anchor, score +3 +100M +83N +3M +48S +100M +51S + + + +collapsed · 7 bp anchor, score +2 with 1 SNV +100M +83N +7M +44S +100M +51S + + +matched (M) +mismatch (SNV) +soft clip (S) +junction (N) + diff --git a/tars/doc/overhang_two_junctions.svg b/tars/doc/overhang_two_junctions.svg new file mode 100644 index 00000000000..f70d683c697 --- /dev/null +++ b/tars/doc/overhang_two_junctions.svg @@ -0,0 +1,77 @@ + +Step 1: more than one splice junction, kept or collapsed +Three real 151 bp two-junction reads, decided at a terminal junction. Bar widths are proportional to base count (large exons capped); introns (N) are drawn as fixed gaps. Kept: chr5:181,237,713, 7M 379N 141M 827N 3M, the trailing 3M matches its far exon (AS +3) so the junction is kept. Kept: chr1:1,049,783, 13M 107N 135M 195N 3M, the trailing 3M mismatches its far exon (CCG vs CCT, AS negative) but reads even worse contiguously (GTC), so the junction is kept. Collapsed: chr1:226,062,810, 2M 2923N 75M 5541N 74M, the leading 2M mismatches its far exon (AG vs GG) but reads through contiguously (AG), so the 2923N collapses and the 2M folds into the 75M giving 77M 5541N 74M. M plus S totals 151 in every case. + + + + +> 1 splice junctions + +BEFORE +AFTER + + + + +kept · 3 bp overhang matches its far exon (AS +3) +7M +379N +141M +827N +3M +7M +379N +141M +827N +3M + + + +kept · 3 bp overhang mismatches far exon, reads worse contiguously +13M +107N +135M +195N +3M +13M +107N +135M +195N +3M + + + +collapsed · 2 bp leading overhang mismatches far exon, reads through +2M +2923N +75M +5541N +74M +77M +5541N +74M + + +matched (M) +mismatch (SNV) +junction (N) + diff --git a/tars/doc/ref_reads_through.svg b/tars/doc/ref_reads_through.svg new file mode 100644 index 00000000000..8e00d69f14d --- /dev/null +++ b/tars/doc/ref_reads_through.svg @@ -0,0 +1,127 @@ + +REF_READS_THROUGH: ref wins over a fabricated tx gap +At one locus, chr8:126,557,042, the tx alignment opens a 54 bp gap to reach an 8 bp far anchor (143M 54N 8M); the ref alt reads through contiguously as 151M with zero mismatches. The read's last 8 bases (GCGGCGGC) match the genome both at the contiguous position and at the downstream exon after the gap, so nothing breaks the tie. The contiguous no-gap read is more parsimonious, so tars keeps ref and drops the tx alt. + + + + + + + + + + + + + + + + + + + +matched (M) + +exon (genome) + +gap (N) + +8 bp anchor + + + +before · two placements at one locus, chr8:126,557,042 + +bwa primary (tx): an 8 bp far anchor across a fabricated 54 bp gap + +143M + + +54N + +8M + +MAPQ 7 + +ref alt (XA): reads through contiguously, no gap, zero mismatches + +151M + + +MAPQ 60 + + +reference genome · chr8 + + + + +exon · reads through contiguously + +126,557,042 +…185 +…239 +downstream exon + + + +the read's last 8 bases match the genome in both places - nothing distinguishes them: +read[144..151] +GCGGCGGC +chr8:126,557,185 (contiguous) +GCGGCGGC +match +chr8:126,557,239 (after 54 bp gap) +GCGGCGGC +match + + +after · ref kept, tx alt dropped + +151M + +MAPQ 60 +both explain the read with zero mismatches; the no-gap read is more parsimonious, so ref wins (NH 1) +shape-based rule (RefFullMatch beats TxHasN); it does not compare mismatch counts + +ref wins · the genome reads through a gap the tx fabricated + diff --git a/tars/doc/ref_vs_tx_decision.svg b/tars/doc/ref_vs_tx_decision.svg new file mode 100644 index 00000000000..dd9a888e625 --- /dev/null +++ b/tars/doc/ref_vs_tx_decision.svg @@ -0,0 +1,81 @@ + +Step 2: transcript vs genome - which placement wins +When a read has both a transcript (spliced, N-cigar) and a genomic alignment, TARS keeps the transcript's junction unless the genome matches cleanly straight through it. Three real cases: genome contiguous at another locus (chr9:86,266,119 73M 5815N 78M tx beats chr2:128,518,864 151M) - transcript wins; genome soft-clips at the boundary (chr1:1,049,917 121M 195N 30M tx beats 115M 36S ref) - transcript wins; genome reads straight through with no gap (chr8:126,557,042 151M ref beats 143M 54N 8M tx) - genome wins, the N was the artifact. + + + +transcript vs genome: does the read splice here? +the transcript's N junction wins unless the genome matches cleanly straight through it - then the N is the artifact + + +genome maps contiguously at another locus +tx chr9:86,266,119 +73M +5815N +78M +ref chr2:128,518,864 +151M +kept in XA +transcript wins + + + + +genome soft-clips at the boundary, cannot cross the intron +tx chr1:1,049,917 +121M +195N +30M +ref chr1:1,049,917 +115M +36S +dropped +transcript wins + + + + +genome matches straight through, no gap +tx chr8:126,557,042 +143M +54N +8M +dropped +ref chr8:126,557,042 +151M +genome wins + + +transcript (M) +genome kept +softclip (S) +intron / gap (N) +dropped + diff --git a/tars/doc/rescue_via_supplementary.svg b/tars/doc/rescue_via_supplementary.svg new file mode 100644 index 00000000000..3243d3843f8 --- /dev/null +++ b/tars/doc/rescue_via_supplementary.svg @@ -0,0 +1,197 @@ + +Step 3: merge supplementary record to primary +Exon 1 here is a novel exon (shown in a deeper blue): it is absent from every transcript contig, so Step 1 cannot translate the read as a single spliced record and bwa instead emits it as two genome records - a short-anchor supplementary (20M in novel exon 1, rest clipped) and a primary (head clipped, 131M in exon 2), complementary across a 166 bp intron. tars merges them into one spliced primary 20M 166N 131M and drops the supplementary; the merged MAPQ is max(primary, supplementary), bumped to 60 when the primary+supplementary pair maps to a single locus. Being novel the junction has no annotated boundary, so it resolves to a canonical GT-AG splice motif - the zoom shows a junction placed 1 bp off being slid onto the motif. + + + + + + + + + + + + + + + + + + + +matched (M) + +softclip (S) + +intron (N) + + + +before · two records for one read + + +reference genome · chr22 + + + +novel exon 1 +exon 2 +166 bp intron + + +supplementary · 20M anchors in novel exon 1, the rest is clipped + +20M + +131S + + +primary · head clipped, 131M anchors in exon 2 + +20S + +131M + + +anchors are complementary; merge once the novel junction resolves to a splice motif + + + + + +after · one spliced primary, supplementary dropped + +20M + + +166N + +131M + +MAPQ 60 +pair maps uniquely + + + +zoom · if the merged junction lands off the boundary, the snap slides it onto the canonical GT-AG motif + + +donor (exon 1 | intron) +acceptor (intron | exon 2) +before +after + + + + + + + + + +T +C +C +G +T +G + + + + + + + + + +T +A +G +T +C +G +CG..TA + + + + + + + + + + +T +C +C +G +T +G + + + + + + + + + + +T +A +G +T +C +G +GT..AG + + +before +20M 166N 131M +junction 1 bp off the motif +after +20M 166N 131M +snapped onto GT..AG + diff --git a/tars/doc/swap_junction_over_contiguous.svg b/tars/doc/swap_junction_over_contiguous.svg new file mode 100644 index 00000000000..962902db327 --- /dev/null +++ b/tars/doc/swap_junction_over_contiguous.svg @@ -0,0 +1,114 @@ + +Swap (JUNCTION_OVER_CONTIGUOUS): spliced placement beats a contiguous one +bwa picks a contiguous 151M placement on chr2:128,518,864 as primary; a transcript alignment on chr9:86,266,119 spans a real 5,815 bp junction (73M 5815N 78M) but bwa demoted it to XA. tars swaps the primary to the spliced chr9 placement, keeps the displaced chr2 placement in XA, drops the lower-scoring chr5 ref alt, and rescues MAPQ 0 to 60. + + + + + + + + + + + + + + + + + + + +matched (M) + +exon (genome) + +intron (N) + + +before · bwa's primary is the contiguous chr2 placement + + +chr2:128,518,864 - bwa primary: contiguous 151M, no junction (a coincidental match at another locus) +reference genome · chr2 + +single exon · no intron here + +151M + +MAPQ 0 + + +chr9:86,266,119 - tx alignment: spans a real 5,815 bp junction (bwa demoted it to XA) +reference genome · chr9 + + + +exon +next exon + +73M + + +5815N + +78M + + +after · primary swapped to the spliced chr9 placement + +73M + + +5815N + +78M + +MAPQ 60 + + +XA: chr2,+128518864,151M,0 +displaced chr2 placement kept in XA (chr5 alt dropped) +a junction across a real intron is stronger evidence than a contiguous match at an unrelated locus (NH 1) + + diff --git a/tars/doc/tail_extension.svg b/tars/doc/tail_extension.svg new file mode 100644 index 00000000000..fa829e68097 --- /dev/null +++ b/tars/doc/tail_extension.svg @@ -0,0 +1,96 @@ + +Step 2: reclaim a trimmed tail +A terminal softclip is walked into the contiguous genome and re-scored. The best-scoring prefix (at least tail_min_extension, up to tail_max_extension) is turned from softclip (S) into matched (M): 131M 20S becomes 151M when the whole clip matches, or 139M 12S when only 8 bp match. CIGAR only, MAPQ unchanged. A walk that stalls where an annotated intron begins is left for rescue. + + + + + + + + + + + + + + + + + + + +matched (M) + +softclip (S) + +reclaimed + + + +before · terminal softclip + +131M + +20S +the softclip is at least 3 bp (tail_min_softclip), so it is re-checked against the genome + + +reference genome · chr16 + + +exon +clip matches + + +walk the clip into the contiguous genome: the best-scoring prefix (3 to 30 bp) becomes M + + + + + +after · read's CIGAR updated to 151M + + +151M +CIGAR only; MAPQ unchanged. Partial match -> the rest stays clipped (e.g. 139M 12S). + diff --git a/tars/doc/tars.png b/tars/doc/tars.png new file mode 100644 index 00000000000..c6f025e108f Binary files /dev/null and b/tars/doc/tars.png differ diff --git a/tars/doc/terminal_micro_junction_collapse.svg b/tars/doc/terminal_micro_junction_collapse.svg new file mode 100644 index 00000000000..60aec630b5d --- /dev/null +++ b/tars/doc/terminal_micro_junction_collapse.svg @@ -0,0 +1,95 @@ + +Step 1: re-check short junctions +A translated read has a 5 bp matched anchor across a 2,000 bp junction. 5 bp is below the 8 bp trust floor (MIN_JUNCTION_ANCHOR), so the junction is re-checked against the reference genome. The genome reads straight through with no gap, so the junction is dropped and the 5 bases become M: 146M 2000N 5M becomes 151M. Had the 5 bases only matched across the intron, the junction would be kept. + + + + + + + + + + + + + + + + + + + +matched (M) + +anchor below 8 bp + +junction (re-checked) + + + +before · translated read, 5 bp anchor across a 2,000 bp junction + +146M + +2000N + +5M +the 5 bp anchor is below the 8 bp floor (MIN_JUNCTION_ANCHOR), so the junction is re-checked + + +reference genome · chr22 + +one continuous exon, no gap + + +re-check the short anchor against the genome: reads straight through = drop the junction + + + + + +after · read's CIGAR updated to 151M + +151M +CIGAR only; MAPQ is set later + diff --git a/tars/doc/translate.svg b/tars/doc/translate.svg new file mode 100644 index 00000000000..8a280a5a4cc --- /dev/null +++ b/tars/doc/translate.svg @@ -0,0 +1,107 @@ + +Step 0: translate the transcript CIGAR to genome +A read aligns to the transcriptome as 151M, where two exons are packed adjacent; it is walked through the exon boundary and an N is inserted where the read crosses it, giving the spliced genomic CIGAR 90M 5000N 61M. CIGAR transform only. + + + + + + + + + + + + + + + + + + + +matched (M) + +exon (genome) + +intron (N) + + + +before · read aligns to the transcriptome + + +exon A +exon B + +151M + +exon boundary the read crosses + + + +insert N at the boundary; matched bases stay M + + +reference genome · chr8 + + + +exon A +exon B +5,000 bp intron (not to scale) + +128,750,000 + +128,755,090 + + +after · spliced genomic read + +90M + + +5000N + +61M +only CIGAR and pos is translated; MAPQ is set later + diff --git a/tars/pom.xml b/tars/pom.xml new file mode 100644 index 00000000000..14c217560cf --- /dev/null +++ b/tars/pom.xml @@ -0,0 +1,79 @@ + + 4.0.0 + + + hmftools + com.hartwig + local-SNAPSHOT + + + tars + jar + ${tars.version} + HMF Tools - Tars + + + + + org.junit + junit-bom + 5.12.0 + pom + import + + + + + + + com.hartwig + hmf-common + + + org.immutables + value + provided + + + + junit + junit + test + + + + com.hartwig + hmf-common + test-jar + test + + + + + + + + src/main/resources + true + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + + true + com.hartwig.hmftools.tars.liftback.TarsApplication + true + true + + + + + + + + diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/common/ContigEntry.java b/tars/src/main/java/com/hartwig/hmftools/tars/common/ContigEntry.java new file mode 100644 index 00000000000..a24460ce350 --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/common/ContigEntry.java @@ -0,0 +1,41 @@ +package com.hartwig.hmftools.tars.common; + +import java.util.List; + +import com.hartwig.hmftools.common.region.BaseRegion; + +// One transcript segment on the alt contig. AltStart/AltEnd are 1-based inclusive positions on contigName; +// exonSpans are 1-based inclusive genomic positions. Strand (+1/-1) is forwarded to set XS:A on spliced reads. +// Annotation-only rows (altStart==0, no contig) carry the genomic exons of transcripts skipped from the FASTA +// (single-exon, all-N) so the liftback exon/junction indexes can be built from the sidecar alone; they are +// excluded from contig-coordinate lifting. +public record ContigEntry( + String contigName, + int altStart, + int altEnd, + String geneId, + String geneName, + String transName, + String chromosome, + int strand, + List exonSpans) +{ + public static final String ANNOTATION_ONLY_CONTIG = "annotation"; + + public static ContigEntry annotationOnly( + final String geneId, final String geneName, final String transName, final String chromosome, + final int strand, final List exonSpans) + { + return new ContigEntry(ANNOTATION_ONLY_CONTIG, 0, 0, geneId, geneName, transName, chromosome, strand, exonSpans); + } + + public boolean isAnnotationOnly() + { + return altStart == 0; + } + + public int contigLength() + { + return altEnd - altStart + 1; + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/common/ContigSidecar.java b/tars/src/main/java/com/hartwig/hmftools/tars/common/ContigSidecar.java new file mode 100644 index 00000000000..9c11c0d2626 --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/common/ContigSidecar.java @@ -0,0 +1,112 @@ +package com.hartwig.hmftools.tars.common; + +import static com.hartwig.hmftools.common.utils.file.FileDelimiters.ITEM_DELIM; +import static com.hartwig.hmftools.tars.common.TarsConfig.TARS_LOGGER; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import com.hartwig.hmftools.common.region.BaseRegion; +import com.hartwig.hmftools.common.utils.file.DelimFileReader; +import com.hartwig.hmftools.common.utils.file.DelimFileWriter; + +public final class ContigSidecar +{ + public enum Column + { + ContigName, + AltStart, + AltEnd, + GeneId, + GeneName, + TransName, + Chromosome, + Strand, + ExonSpans + } + + private static final String SPAN_DELIM = "-"; + + public static void write(final String filename, final List entries) + { + try(DelimFileWriter writer = new DelimFileWriter<>(filename, Column.values(), (entry, row) -> + { + row.set(Column.ContigName, entry.contigName()); + row.set(Column.AltStart, String.valueOf(entry.altStart())); + row.set(Column.AltEnd, String.valueOf(entry.altEnd())); + row.set(Column.GeneId, entry.geneId()); + row.set(Column.GeneName, entry.geneName()); + row.set(Column.TransName, entry.transName()); + row.set(Column.Chromosome, entry.chromosome()); + row.set(Column.Strand, String.valueOf(entry.strand())); + row.set(Column.ExonSpans, entry.exonSpans().stream().map(BaseRegion::toString).collect(Collectors.joining(ITEM_DELIM))); + })) + { + for(ContigEntry entry : entries) + { + writer.writeRow(entry); + } + } + + TARS_LOGGER.info("wrote {} contig entries to {}", entries.size(), filename); + } + + public static List read(final String filename) + { + List entries = new ArrayList<>(); + boolean missingStrandWarned = false; + + try(DelimFileReader reader = new DelimFileReader(filename)) + { + // Strand was added after the initial release; older sidecars lack it. Default to 0 (suppresses XS:A emission) rather than failing. + boolean hasStrand = reader.getColumnNames().contains(Column.Strand.name()); + + for(DelimFileReader.Row row : reader) + { + int strand; + if(hasStrand) + { + strand = Integer.parseInt(row.get(Column.Strand)); + } + else + { + strand = 0; + if(!missingStrandWarned) + { + TARS_LOGGER.warn("contig sidecar {} lacks the Strand column - XS:A:+/- will not " + + "be emitted on tx-derived spliced records. Regenerate the sidecar " + + "with the current SpliceFastaBuilder to enable.", filename); + missingStrandWarned = true; + } + } + + entries.add(new ContigEntry( + row.get(Column.ContigName), + Integer.parseInt(row.get(Column.AltStart)), + Integer.parseInt(row.get(Column.AltEnd)), + row.get(Column.GeneId), + row.get(Column.GeneName), + row.get(Column.TransName), + row.get(Column.Chromosome), + strand, + decodeSpans(row.get(Column.ExonSpans)))); + } + } + + TARS_LOGGER.info("loaded {} contig entries from {}", entries.size(), filename); + return entries; + } + + private static List decodeSpans(final String encoded) + { + List spans = new ArrayList<>(); + for(String token : encoded.split(ITEM_DELIM)) + { + String[] parts = token.split(SPAN_DELIM); + spans.add(new BaseRegion(Integer.parseInt(parts[0]), Integer.parseInt(parts[1]))); + } + return spans; + } + +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/common/SpliceCommon.java b/tars/src/main/java/com/hartwig/hmftools/tars/common/SpliceCommon.java new file mode 100644 index 00000000000..ded3c0d8c10 --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/common/SpliceCommon.java @@ -0,0 +1,11 @@ +package com.hartwig.hmftools.tars.common; + +import static com.hartwig.hmftools.tars.common.TarsConstants.ALT_CONTIG_SUFFIX; + +public final class SpliceCommon +{ + public static String altContigName(final String chromosome) + { + return chromosome + ALT_CONTIG_SUFFIX; + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/common/TarsConfig.java b/tars/src/main/java/com/hartwig/hmftools/tars/common/TarsConfig.java new file mode 100644 index 00000000000..2ed985af7d5 --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/common/TarsConfig.java @@ -0,0 +1,8 @@ +package com.hartwig.hmftools.tars.common; + +import com.hartwig.hmftools.common.logging.HmfLogger; + +public final class TarsConfig +{ + public static final HmfLogger TARS_LOGGER = HmfLogger.getLogger(TarsConfig.class); +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/common/TarsConstants.java b/tars/src/main/java/com/hartwig/hmftools/tars/common/TarsConstants.java new file mode 100644 index 00000000000..bfa747077f0 --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/common/TarsConstants.java @@ -0,0 +1,78 @@ +package com.hartwig.hmftools.tars.common; + +// All Tars constants in one place: tool identity, output file id, the transcript-contig naming scheme, and the +// algorithm/policy thresholds every liftback pass shares. Config-key strings stay with the config class; local +// plumbing constants stay with their owning class. +public final class TarsConstants +{ + // Tool identity. + public static final String APP_NAME = "Tars"; + + // Output file id, e.g. .tars.summary.tsv (mirrors Redux's FILE_ID). + public static final String FILE_ID = "tars"; + + // Transcript-contig naming. An alt contig is the chromosome name plus the _tx suffix (e.g. chr1_tx); the + // delimiter and prefix describe the per-transcript naming used when building the contig FASTA. + public static final String CONTIG_NAME_DELIM = "_"; + public static final String CONTIG_NAME_PREFIX = "ens"; + public static final String ALT_CONTIG_SUFFIX = "_tx"; + + // SpliceFastaBuilder output file ids. + public static final String TRANSCRIPT_CONTIGS_FILE_ID = ".fasta"; + public static final String CONTIG_MAPPINGS_FILE_ID = ".rna_contigs_mappings.tsv"; + + // Junction anchors. + // Minimum M flank (bp) for an N to count as a trusted splice junction. A flank below this carries too + // little evidence to assert a junction (~1/4^8 chance a coincidental anchor matches), so an N with a + // sub-threshold flank is treated as fabricated/untrustworthy: the discriminator won't swap the primary + // onto it (LiftedAlignment.cigarHasRealNJunction). The overhang gate scores terminal anchors against the + // genome instead of applying this length threshold. + public static final int MIN_JUNCTION_ANCHOR = 8; + public static final int ANNOTATED_JUNCTION_MIN_ANCHOR_BP = 1; + public static final int ANNOTATED_JUNCTION_MIN_SOFTCLIP_ANCHOR_BP = 3; + public static final int SPLICE_FLANKING_DELETION_MAX_BP = 5; + + // bwa-mem affine alignment scoring. The single source for every liftback pass that re-scores a boundary: + // the BoundaryReclaim base walk (match/mismatch) and the LiftBackResolver CIGAR reconstruction (all four). + // Soft-clips are not penalised (clipped bases score 0). + public static final int MATCH = 1; + public static final int MISMATCH = -4; + public static final int GAP_OPEN = -6; + public static final int GAP_EXTEND = -1; + + // Overhang gate: keep a terminal splice-junction anchor only if its own affine alignment score (matches +1, + // mismatches -4) at the far exon exceeds this. Equals the bwa soft-clip penalty, so a junction is kept only + // when aligning the anchor scores better than clipping it (a clean 6bp anchor survives, a 5bp anchor does not). + public static final int MIN_OVERHANG_SCORE = 5; + + // Overhang gate length trust: an overhang longer than this is trusted outright and never scored/collapsed - a long + // anchor is a real junction even with a couple of mismatches (SNVs). Only short overhangs (<= this) run the score + // check above. Applies to the terminal-anchor collapse in both the single-junction and multi-junction cases. + public static final int MIN_OVERHANG_LENGTH = 12; + + // MAPQ / AS thresholds. + // Confident placement MAPQ: bumped onto a single-locus MAPQ-0 primary (decidePrimaryMapq) and onto a + // supplementary-resolve merge whose components were all MAPQ 0 (the merge is the placement evidence). + public static final int CONFIDENT_MAPQ = 60; + public static final int PRIMARY_AS_UNMAP_THRESHOLD = 30; + public static final int SUPP_AS_DROP_THRESHOLD = 30; + + // Supplementary-resolve defaults. Annotated-junction policy: min intron 21, max intron 1_000_000. + public static final int DEFAULT_MIN_INTRON_LENGTH = 21; + public static final int DEFAULT_MAX_INTRON_LENGTH = 1_000_000; + + // max supplementaries merged into one read: 1 (typical) or 2 (a middle exon clipped both ends). Deeper chains + // are vanishingly rare (depth 3-4 was 72 reads in a whole sample), so 2 covers essentially all real cases. + public static final int DEFAULT_MAX_SUPP_MERGES = 2; + + // Overlap tolerance (bp) between primary and supp matched regions in a supplementary-resolve merge. BWA primary/supp often + // disagree by 1-5 bases on junction placement; the snap resolves within this window. Compile-time only (not a flag). + public static final int DEFAULT_SOFTCLIP_TOLERANCE = 5; + + // How many over-extended matched bases to trim when probing for an annotated donor/acceptor + // (no-supp ref-verify path). BWA frequently extends past the true exon boundary into the intron; + // measured over-extension caps at ~8 bases. + public static final int DEFAULT_MAX_BOUNDARY_SHIFT = 8; + + private TarsConstants() { } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/fasta/AltContigPacker.java b/tars/src/main/java/com/hartwig/hmftools/tars/fasta/AltContigPacker.java new file mode 100644 index 00000000000..511dde61697 --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/fasta/AltContigPacker.java @@ -0,0 +1,51 @@ +package com.hartwig.hmftools.tars.fasta; + +import static com.hartwig.hmftools.tars.common.SpliceCommon.altContigName; + +import java.util.ArrayList; +import java.util.List; + +import com.hartwig.hmftools.tars.common.ContigEntry; + +// Packs per-transcript sequences into a single per-chromosome alt contig with N-spacers between transcripts. +// Pure function - no I/O. +public final class AltContigPacker +{ + private final String mSpacer; + + public AltContigPacker(final int spacerLength) + { + mSpacer = "N".repeat(spacerLength); + } + + public PackResult pack(final String chromosome, final List transcripts) + { + String altContig = altContigName(chromosome); + StringBuilder sequence = new StringBuilder(); + List entries = new ArrayList<>(); + + for(final TranscriptContigBuilder.TranscriptContigResult transcript : transcripts) + { + if(sequence.length() > 0) + { + sequence.append(mSpacer); + } + + int altStart = sequence.length() + 1; + sequence.append(transcript.sequence()); + int altEnd = sequence.length(); + + entries.add(new ContigEntry( + altContig, altStart, altEnd, + transcript.geneId(), transcript.geneName(), transcript.transName(), transcript.chromosome(), + transcript.strand(), transcript.exonSpans())); + } + + return new PackResult(altContig, sequence.toString(), entries); + } + + public record PackResult(String altContig, String sequence, List entries) + { + public boolean isEmpty() { return entries.isEmpty(); } + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/fasta/SpliceFastaBuilder.java b/tars/src/main/java/com/hartwig/hmftools/tars/fasta/SpliceFastaBuilder.java new file mode 100644 index 00000000000..bf21229ffe5 --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/fasta/SpliceFastaBuilder.java @@ -0,0 +1,196 @@ +package com.hartwig.hmftools.tars.fasta; + +import static com.hartwig.hmftools.common.perf.PerformanceCounter.runTimeMinsStr; +import static com.hartwig.hmftools.common.utils.file.FileWriterUtils.createBufferedWriter; +import static com.hartwig.hmftools.tars.common.TarsConstants.CONTIG_MAPPINGS_FILE_ID; +import static com.hartwig.hmftools.tars.common.TarsConstants.TRANSCRIPT_CONTIGS_FILE_ID; +import static com.hartwig.hmftools.tars.common.TarsConstants.APP_NAME; +import static com.hartwig.hmftools.tars.common.TarsConfig.TARS_LOGGER; + +import java.io.BufferedWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Map; + +import com.hartwig.hmftools.common.ensemblcache.EnsemblDataCache; +import com.hartwig.hmftools.common.gene.ExonData; +import com.hartwig.hmftools.common.gene.GeneData; +import com.hartwig.hmftools.common.gene.TranscriptData; +import com.hartwig.hmftools.common.genome.refgenome.RefGenomeSource; +import com.hartwig.hmftools.common.region.BaseRegion; +import com.hartwig.hmftools.common.utils.config.ConfigBuilder; +import com.hartwig.hmftools.tars.common.ContigEntry; +import com.hartwig.hmftools.tars.common.ContigSidecar; + +public class SpliceFastaBuilder +{ + // N-padding between transcripts on the same alt contig + private static final int SPACER_LENGTH = 500; + + private final SpliceFastaConfig mConfig; + private final EnsemblDataCache mEnsemblDataCache; + + public SpliceFastaBuilder(final ConfigBuilder configBuilder) + { + mConfig = new SpliceFastaConfig(configBuilder); + mEnsemblDataCache = new EnsemblDataCache(configBuilder); + } + + public void run() + { + long startTimeMs = System.currentTimeMillis(); + + RefGenomeSource refGenome = RefGenomeSource.loadRefGenome(mConfig.RefGenomeFile); + if(refGenome == null) + { + TARS_LOGGER.error("failed to load ref genome: {}", mConfig.RefGenomeFile); + System.exit(1); + } + + TARS_LOGGER.info("loading ensembl cache from {}", mConfig.EnsemblDataDir); + mEnsemblDataCache.setRequiredData(true, false, false, false); + mEnsemblDataCache.load(false); + + Map> chrGeneMap = mEnsemblDataCache.getChrGeneDataMap(); + + String fastaFile = mConfig.formFilename(TRANSCRIPT_CONTIGS_FILE_ID); + String mappingsFile = mConfig.formFilename(CONTIG_MAPPINGS_FILE_ID); + + TranscriptContigBuilder builder = new TranscriptContigBuilder(refGenome); + AltContigPacker packer = new AltContigPacker(SPACER_LENGTH); + + List contigEntries = new ArrayList<>(); + + int genesProcessed = 0; + int contigsWritten = 0; + int skippedSingleExon = 0; + int skippedAllN = 0; + + try(BufferedWriter fastaWriter = createBufferedWriter(fastaFile)) + { + for(Map.Entry> chrEntry : chrGeneMap.entrySet()) + { + String chromosome = chrEntry.getKey(); + List chromosomeTranscripts = new ArrayList<>(); + + for(GeneData gene : chrEntry.getValue()) + { + ++genesProcessed; + + List transcripts = mEnsemblDataCache.getTranscripts(gene.GeneId); + if(transcripts == null || transcripts.isEmpty()) + { + throw new IllegalStateException("ensembl cache returned no transcripts for gene " + gene.GeneId); + } + + for(TranscriptData transcript : transcripts) + { + // single-exon transcripts have no junctions, so no contig; still record their exon as an + // annotation-only row so the liftback exon index matches the full ensembl cache. + if(transcript.exons().size() < 2) + { + ++skippedSingleExon; + contigEntries.add(annotationEntry(gene, transcript, exonSpansOf(transcript))); + continue; + } + + TranscriptContigBuilder.TranscriptContigResult result = builder.build(gene, transcript); + if(result == null) + continue; + + // all-N contig is alignment-useless, so no contig; keep its exons as annotation only. + if(isAllN(result.sequence())) + { + ++skippedAllN; + contigEntries.add(annotationEntry(gene, transcript, result.exonSpans())); + continue; + } + + chromosomeTranscripts.add(result); + } + + if(genesProcessed % 1000 == 0) + { + TARS_LOGGER.debug("processed {} genes", genesProcessed); + } + } + + if(chromosomeTranscripts.isEmpty()) + continue; + + AltContigPacker.PackResult packed = packer.pack(chromosome, chromosomeTranscripts); + writeFastaContig(fastaWriter, packed.altContig(), packed.sequence()); + contigEntries.addAll(packed.entries()); + contigsWritten += packed.entries().size(); + } + } + catch(IOException e) + { + TARS_LOGGER.error("failed to write transcript contigs FASTA: {}", e.toString()); + System.exit(1); + } + + TARS_LOGGER.info("wrote {} transcript contigs to {} (skipped {} single-exon, {} all-N)", + contigsWritten, fastaFile, skippedSingleExon, skippedAllN); + + ContigSidecar.write(mappingsFile, contigEntries); + + TARS_LOGGER.info("next: cat {} > ref_genome__rna_contigs.fasta && bwa-mem2 index ref_genome__rna_contigs.fasta", + fastaFile); + + TARS_LOGGER.info("SpliceFastaBuilder complete, mins({})", runTimeMinsStr(startTimeMs)); + } + + private static ContigEntry annotationEntry(final GeneData gene, final TranscriptData transcript, final List exonSpans) + { + return ContigEntry.annotationOnly( + gene.GeneId, gene.GeneName, transcript.TransName, gene.Chromosome, gene.Strand, exonSpans); + } + + private static List exonSpansOf(final TranscriptData transcript) + { + List ordered = new ArrayList<>(transcript.exons()); + ordered.sort(Comparator.comparingInt(exon -> exon.Start)); + + List spans = new ArrayList<>(ordered.size()); + for(final ExonData exon : ordered) + { + spans.add(new BaseRegion(exon.Start, exon.End)); + } + return spans; + } + + private static boolean isAllN(final String sequence) + { + for(int i = 0; i < sequence.length(); ++i) + { + char c = sequence.charAt(i); + if(c != 'N' && c != 'n') + { + return false; + } + } + return true; + } + + private static void writeFastaContig(final BufferedWriter writer, final String contigName, final String sequence) throws IOException + { + writer.write(">" + contigName); + writer.newLine(); + writer.write(sequence); + writer.newLine(); + } + + public static void main(final String[] args) + { + ConfigBuilder configBuilder = new ConfigBuilder(APP_NAME); + SpliceFastaConfig.addConfig(configBuilder); + + configBuilder.checkAndParseCommandLine(args); + + SpliceFastaBuilder builder = new SpliceFastaBuilder(configBuilder); + builder.run(); + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/fasta/SpliceFastaConfig.java b/tars/src/main/java/com/hartwig/hmftools/tars/fasta/SpliceFastaConfig.java new file mode 100644 index 00000000000..4bfc9241764 --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/fasta/SpliceFastaConfig.java @@ -0,0 +1,61 @@ +package com.hartwig.hmftools.tars.fasta; + +import static com.hartwig.hmftools.common.ensemblcache.EnsemblDataCache.ENSEMBL_DATA_DIR; +import static com.hartwig.hmftools.common.ensemblcache.EnsemblDataCache.addEnsemblDir; +import static com.hartwig.hmftools.common.genome.refgenome.RefGenomeSource.REF_GENOME; +import static com.hartwig.hmftools.common.genome.refgenome.RefGenomeSource.addRefGenomeFile; +import static com.hartwig.hmftools.common.genome.refgenome.RefGenomeSource.addRefGenomeVersion; +import static com.hartwig.hmftools.common.genome.refgenome.RefGenomeVersion.REF_GENOME_VERSION; +import static com.hartwig.hmftools.common.genome.refgenome.RefGenomeVersion.from; +import static com.hartwig.hmftools.common.utils.config.ConfigUtils.addLoggingOptions; +import static com.hartwig.hmftools.common.utils.file.FileWriterUtils.OUTPUT_ID; +import static com.hartwig.hmftools.common.utils.file.FileWriterUtils.addOutputOptions; +import static com.hartwig.hmftools.common.utils.file.FileWriterUtils.checkCreateOutputDir; +import static com.hartwig.hmftools.common.utils.file.FileWriterUtils.parseOutputDir; + +import com.hartwig.hmftools.common.genome.refgenome.RefGenomeVersion; +import com.hartwig.hmftools.common.utils.config.ConfigBuilder; + +public class SpliceFastaConfig +{ + public final String EnsemblDataDir; + public final String RefGenomeFile; + public final RefGenomeVersion RefGenVersion; + public final String OutputDir; + public final String OutputId; + + public SpliceFastaConfig(final ConfigBuilder configBuilder) + { + EnsemblDataDir = configBuilder.getValue(ENSEMBL_DATA_DIR); + RefGenomeFile = configBuilder.getValue(REF_GENOME); + RefGenVersion = configBuilder.hasValue(REF_GENOME_VERSION) ? from(configBuilder) : RefGenomeVersion.V38; + OutputDir = parseOutputDir(configBuilder); + OutputId = configBuilder.getValue(OUTPUT_ID); + + if(OutputDir == null) + { + throw new IllegalArgumentException("missing required config: output_dir"); + } + + if(!checkCreateOutputDir(OutputDir)) + { + throw new IllegalStateException("failed to create output directory: " + OutputDir); + } + } + + public String formFilename(final String fileId) + { + String prefix = OutputId != null ? OutputId : "ref_genome_v" + RefGenVersion.identifier() + "_rna_contigs"; + return OutputDir + prefix + fileId; + } + + public static void addConfig(final ConfigBuilder configBuilder) + { + addEnsemblDir(configBuilder, true); + addRefGenomeFile(configBuilder, true); + addRefGenomeVersion(configBuilder); + + addOutputOptions(configBuilder); + addLoggingOptions(configBuilder); + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/fasta/TranscriptContigBuilder.java b/tars/src/main/java/com/hartwig/hmftools/tars/fasta/TranscriptContigBuilder.java new file mode 100644 index 00000000000..e8a5b737206 --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/fasta/TranscriptContigBuilder.java @@ -0,0 +1,67 @@ +package com.hartwig.hmftools.tars.fasta; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + +import com.hartwig.hmftools.common.gene.ExonData; +import com.hartwig.hmftools.common.gene.GeneData; +import com.hartwig.hmftools.common.gene.TranscriptData; +import com.hartwig.hmftools.common.genome.refgenome.RefGenomeInterface; +import com.hartwig.hmftools.common.region.BaseRegion; + +// Concatenates a transcript's exons into a forward-strand sequence with genomic span metadata. +// Placement onto the alt contig (altStart/altEnd) is SpliceFastaBuilder's responsibility. +public class TranscriptContigBuilder +{ + private final RefGenomeInterface mRefGenome; + + public TranscriptContigBuilder(final RefGenomeInterface refGenome) + { + mRefGenome = refGenome; + } + + public TranscriptContigResult build(final GeneData gene, final TranscriptData transcript) + { + if(transcript.exons() == null || transcript.exons().isEmpty()) + { + return null; + } + + // exons are stored by transcription rank; re-sort by genomic start for forward-strand orientation + List ordered = new ArrayList<>(transcript.exons()); + ordered.sort(Comparator.comparingInt(e -> e.Start)); + + List spans = new ArrayList<>(ordered.size()); + StringBuilder sequence = new StringBuilder(); + int prevEnd = -1; + for(ExonData exon : ordered) + { + if(exon.Start <= prevEnd) + { + throw new IllegalStateException(String.format( + "overlapping exons in transcript %s gene %s: exon [%d,%d] overlaps prior exon ending at %d", + transcript.TransName, gene.GeneId, exon.Start, exon.End, prevEnd)); + } + + spans.add(new BaseRegion(exon.Start, exon.End)); + sequence.append(mRefGenome.getBaseString(gene.Chromosome, exon.Start, exon.End)); + prevEnd = exon.End; + } + + return new TranscriptContigResult( + gene.GeneId, gene.GeneName, transcript.TransName, gene.Chromosome, gene.Strand, + spans, sequence.toString()); + } + + public record TranscriptContigResult( + String geneId, + String geneName, + String transName, + String chromosome, + int strand, + List exonSpans, + String sequence) + { + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/AlignmentScorer.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/AlignmentScorer.java new file mode 100644 index 00000000000..dd5d8c2d6a8 --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/AlignmentScorer.java @@ -0,0 +1,83 @@ +package com.hartwig.hmftools.tars.liftback; + +import com.hartwig.hmftools.tars.common.TarsConstants; +import com.hartwig.hmftools.tars.liftback.supplementary.RefSequenceSource; + +import htsjdk.samtools.Cigar; +import htsjdk.samtools.CigarElement; +import htsjdk.samtools.TextCigarCodec; + +// bwa-mem-style score of a lifted alignment against the reference genome: match/mismatch over aligned bases, an +// affine gap for each I/D, introns (N) and clips (S/H) free. bwa's own AlignmentScore is scored in tx-contig space +// for a transcript candidate and is absent for XA alts, so it cannot rank a read's candidate placements against +// each other; this recomputes every candidate on one comparable genomic scale for the Step 2 tie-break. +public final class AlignmentScorer +{ + private AlignmentScorer() { } + + // readBases must already be in the alignment's own orientation (reverse-complemented for an opposite-strand alt). + // Returns Integer.MIN_VALUE when the reference is unavailable, matching LiftedAlignment's "not computed" sentinel. + public static int score( + final RefSequenceSource refSource, final String chromosome, final int alignmentStart, + final String cigarStr, final byte[] readBases) + { + if(refSource == null || readBases == null || cigarStr == null) + { + return Integer.MIN_VALUE; + } + + Cigar cigar = TextCigarCodec.decode(cigarStr); + int score = 0; + int queryPos = 0; + int refPos = alignmentStart; + + for(final CigarElement element : cigar.getCigarElements()) + { + int length = element.getLength(); + switch(element.getOperator()) + { + case M: + case EQ: + case X: + byte[] ref = refSource.getBases(chromosome, refPos, refPos + length - 1); + for(int i = 0; i < length; ++i) + { + boolean match = ref != null && i < ref.length && queryPos + i < readBases.length + && basesEqualIgnoreCase(readBases[queryPos + i], ref[i]); + score += match ? TarsConstants.MATCH : TarsConstants.MISMATCH; + } + queryPos += length; + refPos += length; + break; + + case I: + score += TarsConstants.GAP_OPEN + (length - 1) * TarsConstants.GAP_EXTEND; + queryPos += length; + break; + + case D: + score += TarsConstants.GAP_OPEN + (length - 1) * TarsConstants.GAP_EXTEND; + refPos += length; + break; + + case N: + refPos += length; + break; + + case S: + queryPos += length; + break; + + default: // H, P consume neither aligned query nor scored reference + break; + } + } + + return score; + } + + private static boolean basesEqualIgnoreCase(final byte a, final byte b) + { + return (a & ~0x20) == (b & ~0x20); + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/ChunkProducer.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/ChunkProducer.java new file mode 100644 index 00000000000..d81d2e33bb9 --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/ChunkProducer.java @@ -0,0 +1,63 @@ +package com.hartwig.hmftools.tars.liftback; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.atomic.LongAdder; + +import htsjdk.samtools.SAMRecord; + +// Cuts a name-grouped record stream into chunks of whole read-name groups, the unit handed to each worker. +// A chunk is flushed only at a name boundary, so a fragment's records are never split across chunks (which +// would defeat the per-group cache and orphan mates/supps). Used by ShardedChunkProducer's per-shard readers. +public class ChunkProducer +{ + private ChunkProducer() { } + + // reference-compared sentinel enqueued (one per worker) to signal end-of-stream. + public static final List END_OF_STREAM = new ArrayList<>(); + + // sink that may block (the bounded queue) and so propagates InterruptedException. + interface ChunkSink + { + void accept(List chunk) throws InterruptedException; + } + + // cut the stream into chunks of >= targetReads, never splitting a name group. readsCounter (nullable) is + // bumped by each chunk's size as it is flushed, so a shared monitor can report aggregate throughput. + static void streamChunks( + final Iterator iter, final int targetReads, final ChunkSink sink, final LongAdder readsCounter) + throws InterruptedException + { + List chunk = new ArrayList<>(targetReads); + String currentName = null; + + while(iter.hasNext()) + { + SAMRecord record = iter.next(); + String name = record.getReadName(); + + if(currentName != null && !name.equals(currentName) && chunk.size() >= targetReads) + { + if(readsCounter != null) + { + readsCounter.add(chunk.size()); + } + sink.accept(chunk); + chunk = new ArrayList<>(targetReads); + } + + chunk.add(record); + currentName = name; + } + + if(!chunk.isEmpty()) + { + if(readsCounter != null) + { + readsCounter.add(chunk.size()); + } + sink.accept(chunk); + } + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/ContigTranslator.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/ContigTranslator.java new file mode 100644 index 00000000000..ad5fa2b149f --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/ContigTranslator.java @@ -0,0 +1,460 @@ +package com.hartwig.hmftools.tars.liftback; + +import static com.hartwig.hmftools.tars.common.TarsConstants.SPLICE_FLANKING_DELETION_MAX_BP; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import com.hartwig.hmftools.common.region.BaseRegion; +import com.hartwig.hmftools.tars.common.ContigEntry; + +import htsjdk.samtools.Cigar; +import htsjdk.samtools.CigarElement; +import htsjdk.samtools.CigarOperator; + +public final class ContigTranslator +{ + public static TranslationResult translate(final ContigEntry contig, final int contigPos, final Cigar contigCigar) + { + List spans = contig.exonSpans(); + if(spans.isEmpty() || contigPos > contig.altEnd()) + { + return null; + } + + // reclaim any M bases that over-run the transcript bounds into soft-clip so the read still lifts. + ClampedAlignment clamped = clampToContigBounds(contig, contigPos, contigCigar); + if(clamped == null) + { + return null; + } + + // locate the exon span (and its genomic position) that the alignment starts in. + int localPos = clamped.contigPos() - contig.altStart() + 1; + SpanLocation start = locateStartSpan(spans, localPos); + if(start == null) + { + return null; // read starts past the end of the contig + } + + return walkCigarToGenome(contig, spans, start, clamped.cigar()); + } + + // bwa-mem2 can anchor into the alt-contig spacer-N region on either side; convert the overhanging M + // bases to soft-clip so the read still lifts. Returns null when the overhang can't be absorbed. + private static ClampedAlignment clampToContigBounds(final ContigEntry contig, int contigPos, Cigar cigar) + { + if(contigPos < contig.altStart()) + { + int overhang = contig.altStart() - contigPos; + cigar = clampLeadingMToSoftClip(cigar, overhang); + if(cigar == null) + { + return null; + } + contigPos = contig.altStart(); + } + + int readEnd = contigPos + cigar.getReferenceLength() - 1; + if(readEnd > contig.altEnd()) + { + int overhang = readEnd - contig.altEnd(); + cigar = clampTrailingMToSoftClip(cigar, overhang); + if(cigar == null) + { + return null; + } + } + + return new ClampedAlignment(cigar, contigPos); + } + + // Finds the exon span containing localPos (1-based offset into the concatenated exons) and the genomic + // position there. Returns null when localPos runs past the last exon. + private static SpanLocation locateStartSpan(final List spans, final int localPos) + { + int exonLengthSoFar = 0; + for(int i = 0; i < spans.size(); ++i) + { + BaseRegion span = spans.get(i); + if(localPos <= exonLengthSoFar + span.baseLength()) + { + int genomicPos = span.start() + (localPos - exonLengthSoFar - 1); + return new SpanLocation(i, genomicPos); + } + exonLengthSoFar += span.baseLength(); + } + return null; + } + + // Walks the clamped contig-space CIGAR, emitting an N at every exon boundary crossed to produce a + // genome-spaced CIGAR plus the introns it implied. Returns null when the read extends past the last exon. + private static TranslationResult walkCigarToGenome( + final ContigEntry contig, final List spans, final SpanLocation start, final Cigar cigar) + { + int genomicStart = start.genomicPos(); + int currentSpanIndex = start.spanIndex(); + int currentGenomicPos = start.genomicPos(); + + List outElements = new ArrayList<>(); + List impliedIntrons = new ArrayList<>(); + + for(final CigarElement element : cigar.getCigarElements()) + { + CigarOperator op = element.getOperator(); + int remaining = element.getLength(); + + if(!op.consumesReferenceBases()) + { + outElements.add(new CigarElement(remaining, op)); + continue; + } + + while(remaining > 0) + { + BaseRegion currentSpan = spans.get(currentSpanIndex); + + if(currentGenomicPos > currentSpan.end()) + { + if(currentSpanIndex + 1 >= spans.size()) + { + return null; // read extends past last span + } + + BaseRegion nextSpan = spans.get(currentSpanIndex + 1); + int intronStart = currentSpan.end() + 1; + int intronEnd = nextSpan.start() - 1; + + if(intronEnd >= intronStart) + { + outElements.add(new CigarElement(intronEnd - intronStart + 1, CigarOperator.N)); + impliedIntrons.add(new BaseRegion(intronStart, intronEnd)); + } + + ++currentSpanIndex; + currentSpan = nextSpan; + currentGenomicPos = currentSpan.start(); + } + + int remainInSpan = currentSpan.end() - currentGenomicPos + 1; + int take = Math.min(remaining, remainInSpan); + + outElements.add(new CigarElement(take, op)); + currentGenomicPos += take; + remaining -= take; + } + } + + return new TranslationResult( + contig.chromosome(), genomicStart, + new Cigar(mergeAdjacentSameOp(dropZeroLength(collapseSpliceFlankingDeletions(outElements)))), + impliedIntrons); + } + + static List dropZeroLength(final List elements) + { + List result = new ArrayList<>(elements.size()); + + for(final CigarElement element : elements) + { + if(element.getLength() > 0) + result.add(element); + } + + return result; + } + + static List mergeAdjacentSameOp(final List elements) + { + List merged = new ArrayList<>(elements.size()); + + for(final CigarElement element : elements) + { + if(!merged.isEmpty() && merged.get(merged.size() - 1).getOperator() == element.getOperator()) + { + CigarElement prev = merged.remove(merged.size() - 1); + merged.add(new CigarElement(prev.getLength() + element.getLength(), element.getOperator())); + } + else + { + merged.add(element); + } + } + + return merged; + } + + // (cigar, contigPos) after the spacer-overhang clamp; contigPos advances when the leading edge is clipped. + private record ClampedAlignment(Cigar cigar, int contigPos) + { + } + + // exon span index + genomic position where an alignment begins. + private record SpanLocation(int spanIndex, int genomicPos) + { + } + + // A D straddling an exon boundary lifts as xD nN yD - artifact of tx-FASTA off-by-N at the junction. + // Absorb small flanking Ds into the N (preserves read and ref span); keep larger ones as real deletions. + static List collapseSpliceFlankingDeletions(final List elements) + { + if(elements.size() < 3) + { + return elements; + } + + List result = new ArrayList<>(elements.size()); + for(int i = 0; i < elements.size(); ++i) + { + CigarElement element = elements.get(i); + + if(element.getOperator() != CigarOperator.N) + { + result.add(element); + continue; + } + + int splicedLength = element.getLength(); + + if(!result.isEmpty() && isAbsorbableDeletion(result.get(result.size() - 1))) + { + splicedLength += result.remove(result.size() - 1).getLength(); + } + + while(i + 1 < elements.size() && isAbsorbableDeletion(elements.get(i + 1))) + { + splicedLength += elements.get(++i).getLength(); + } + + result.add(new CigarElement(splicedLength, CigarOperator.N)); + } + return result; + } + + private static boolean isAbsorbableDeletion(final CigarElement element) + { + return element.getOperator() == CigarOperator.D && element.getLength() <= SPLICE_FLANKING_DELETION_MAX_BP; + } + + public static final class MicroAnchorResult + { + public final Cigar AdjustedCigar; + public final int StartShift; // bases the alignment start advanced (from a leading trim) + + MicroAnchorResult(final Cigar cigar, final int startShift) + { + AdjustedCigar = cigar; + StartShift = startShift; + } + } + + // Drops fabricated sub-threshold junction anchors at both ends. A tx-contig walk that dribbles past + // an exon boundary leaves a tiny "yM nN..." or "...nN yM zS" anchor; a junction whose flanking block + // is shorter than the overhang floor is unsupported and rejected. Two floors: bareAnchorFloor for a read + // genuinely starting inside an exon (no adjacent S, kept down to 1bp); softclipAnchorFloor for + // an anchor sitting next to a softclip (bwa over-ran the boundary, so the implied junction is + // unsupported - roll the yM into the softclip). Leading trim advances start past dropped yM+intron. + public static MicroAnchorResult trimMicroAnchors( + final Cigar cigar, final int bareAnchorFloor, final int softclipAnchorFloor) + { + List elements = new ArrayList<>(cigar.getCigarElements()); + elements = trimTrailingAnchor(elements, softclipAnchorFloor); // trailing trim is always softclip-adjacent + int startShift = 0; + + boolean hasLeadingS = !elements.isEmpty() && elements.get(0).getOperator() == CigarOperator.S; + int leadingFloor = hasLeadingS ? softclipAnchorFloor : bareAnchorFloor; + if(leadingAnchorTrimmable(elements, leadingFloor)) + { + int anchorIdx = hasLeadingS ? 1 : 0; + int existingS = hasLeadingS ? elements.get(0).getLength() : 0; + int tinyAnchor = elements.get(anchorIdx).getLength(); + int intron = elements.get(anchorIdx + 1).getLength(); + startShift = tinyAnchor + intron; + List trimmed = new ArrayList<>(elements.size() - anchorIdx - 1); + trimmed.add(new CigarElement(existingS + tinyAnchor, CigarOperator.S)); + for(int i = anchorIdx + 2; i < elements.size(); ++i) + { + trimmed.add(elements.get(i)); + } + elements = trimmed; + } + + return new MicroAnchorResult(new Cigar(elements), startShift); + } + + public static Cigar trimTrailingMicroAnchor(final Cigar cigar, final int minAnchorBp) + { + return new Cigar(trimTrailingAnchor(new ArrayList<>(cigar.getCigarElements()), minAnchorBp)); + } + + private static List trimTrailingAnchor(final List elements, final int minAnchorBp) + { + int last = elements.size() - 1; + if(last < 2) + { + return elements; + } + if(elements.get(last).getOperator() != CigarOperator.S) + { + return elements; + } + CigarElement tailMatch = elements.get(last - 1); + if(tailMatch.getOperator() != CigarOperator.M || tailMatch.getLength() >= minAnchorBp) + { + return elements; + } + if(elements.get(last - 2).getOperator() != CigarOperator.N) + { + return elements; + } + // without a preceding M the trimmed cigar would start with S/N - refuse + boolean hasAnchorBeforeIntron = false; + for(int i = 0; i < last - 2; ++i) + { + if(elements.get(i).getOperator() == CigarOperator.M) + { + hasAnchorBeforeIntron = true; + break; + } + } + if(!hasAnchorBeforeIntron) + { + return elements; + } + + List result = new ArrayList<>(last - 1); + for(int i = 0; i < last - 2; ++i) + { + result.add(elements.get(i)); + } + result.add(new CigarElement(tailMatch.getLength() + elements.get(last).getLength(), CigarOperator.S)); + return result; + } + + // "[zS]? yM nN <...>M<...>" - leading S optional (absent when source starts with M, no clip to merge) + private static boolean leadingAnchorTrimmable(final List elements, final int minAnchorBp) + { + if(elements.isEmpty()) + { + return false; + } + boolean hasLeadingS = elements.get(0).getOperator() == CigarOperator.S; + int anchorIdx = hasLeadingS ? 1 : 0; + if(anchorIdx + 2 >= elements.size()) + { + return false; + } + CigarElement tinyAnchor = elements.get(anchorIdx); + if(tinyAnchor.getOperator() != CigarOperator.M || tinyAnchor.getLength() >= minAnchorBp) + { + return false; + } + if(elements.get(anchorIdx + 1).getOperator() != CigarOperator.N) + { + return false; + } + for(int i = anchorIdx + 2; i < elements.size(); ++i) + { + if(elements.get(i).getOperator() == CigarOperator.M) + { + return true; + } + } + return false; + } + + // true if a leading or trailing S abuts an interior exon boundary (not the outermost end of the first/last exon) + public static boolean hasSoftClipAtExonBoundary( + final ContigEntry contig, final int contigPos, final Cigar contigCigar) + { + List spans = contig.exonSpans(); + if(spans.size() < 2 || contigCigar.isEmpty()) + { + return false; + } + + List elements = contigCigar.getCigarElements(); + boolean leadingSoftClip = elements.get(0).getOperator() == CigarOperator.S; + boolean trailingSoftClip = elements.get(elements.size() - 1).getOperator() == CigarOperator.S; + if(!leadingSoftClip && !trailingSoftClip) + { + return false; + } + + int localPos = contigPos - contig.altStart() + 1; + int endLocalPos = localPos + contigCigar.getReferenceLength() - 1; + + int exonLengthSoFar = 0; + for(int i = 0; i < spans.size() - 1; ++i) + { + exonLengthSoFar += spans.get(i).baseLength(); + if(leadingSoftClip && localPos == exonLengthSoFar + 1) + { + return true; + } + if(trailingSoftClip && endLocalPos == exonLengthSoFar) + { + return true; + } + } + return false; + } + + // converts the first `overhang` M bases to soft-clip, merging any existing leading S; null if not possible + private static Cigar clampLeadingMToSoftClip(final Cigar cigar, final int overhang) + { + List elements = cigar.getCigarElements(); + int existingLeadingSoftClip = 0; + int idx = 0; + while(idx < elements.size() && elements.get(idx).getOperator() == CigarOperator.S) + { + existingLeadingSoftClip += elements.get(idx).getLength(); + ++idx; + } + if(idx >= elements.size()) + { + return null; + } + + CigarElement first = elements.get(idx); + if(first.getOperator() != CigarOperator.M || first.getLength() <= overhang) + { + return null; + } + + List out = new ArrayList<>(elements.size()); + out.add(new CigarElement(existingLeadingSoftClip + overhang, CigarOperator.S)); + out.add(new CigarElement(first.getLength() - overhang, CigarOperator.M)); + for(int i = idx + 1; i < elements.size(); ++i) + { + out.add(elements.get(i)); + } + return new Cigar(out); + } + + // trailing-edge mirror of clampLeadingMToSoftClip: reverse the elements, clamp the (now-leading) edge, reverse back. + private static Cigar clampTrailingMToSoftClip(final Cigar cigar, final int overhang) + { + List reversed = new ArrayList<>(cigar.getCigarElements()); + Collections.reverse(reversed); + + Cigar clamped = clampLeadingMToSoftClip(new Cigar(reversed), overhang); + if(clamped == null) + { + return null; + } + + List out = new ArrayList<>(clamped.getCigarElements()); + Collections.reverse(out); + return new Cigar(out); + } + + public record TranslationResult( + String chromosome, + int genomicStart, + Cigar genomicCigar, + List impliedIntrons) + { + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/DecidingFeature.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/DecidingFeature.java new file mode 100644 index 00000000000..32b6e0cb91d --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/DecidingFeature.java @@ -0,0 +1,26 @@ +package com.hartwig.hmftools.tars.liftback; + +// Why the primary was resolved the way it was - the evidence that decided ref vs tx. Null for the +// non-decision record states (unmapped / lift-failed / supplementary), which carry no contest. +public enum DecidingFeature +{ + SOLE_REF, // only ref aligned + SOLE_TX, // only tx aligned + CONCORDANT, // ref and tx agree, both contiguous, no junction + REF_READS_THROUGH, // ref reads contiguously where tx carries a junction - read is unspliced + JUNCTION, // tx spans a trusted junction the ref cannot - tx wins + JUNCTION_OVER_CONTIGUOUS, // multi-locus: tx junction beats intronless ref alts + MULTIMAPPER, // multi-locus, no junction to break the tie + AMBIGUOUS; // single locus, no rule fires + + // The outcome is a strict function of the feature, so it is derived here rather than stored alongside. + public Outcome outcome() + { + return switch(this) + { + case SOLE_REF, CONCORDANT, REF_READS_THROUGH -> Outcome.REF; + case SOLE_TX, JUNCTION, JUNCTION_OVER_CONTIGUOUS -> Outcome.TX; + case MULTIMAPPER, AMBIGUOUS -> Outcome.UNRESOLVED; + }; + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/ExcludedRegions.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/ExcludedRegions.java new file mode 100644 index 00000000000..03dbd7c67e9 --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/ExcludedRegions.java @@ -0,0 +1,111 @@ +package com.hartwig.hmftools.tars.liftback; + +import static com.hartwig.hmftools.common.region.ChrBaseRegion.getChromosomeFieldIndex; +import static com.hartwig.hmftools.common.region.ChrBaseRegion.getPositionEndFieldIndex; +import static com.hartwig.hmftools.common.region.ChrBaseRegion.getPositionStartFieldIndex; +import static com.hartwig.hmftools.common.utils.file.FileDelimiters.inferFileDelimiter; +import static com.hartwig.hmftools.common.utils.file.FileReaderUtils.createFieldsIndexMap; +import static com.hartwig.hmftools.tars.common.TarsConfig.TARS_LOGGER; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Comparator; +import java.util.List; +import java.util.Map; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.hartwig.hmftools.common.region.ChrBaseRegion; + +// Genomic regions reads are excluded from (e.g. RNA rRNA / 7SL / acrocentric contamination zones). Loaded from a +// regions TSV (Chromosome/PosStart/PosEnd) and queried post-lift on lifted genomic coords (excludes()): a primary +// lifting in is unmapped, a supp dropped. Post-lift because a tx-contig read's input coords are chrN_tx and can't +// be tested against the genomic region list; some excluded zones (acrocentric arms) ARE in the transcriptome. +// Per-chromosome regions are sorted for an O(log N) overlap test. +public class ExcludedRegions +{ + private final Map> mRegionsByChr; + + ExcludedRegions(final Map> regionsByChr) + { + mRegionsByChr = regionsByChr; + for(final List regions : mRegionsByChr.values()) + { + regions.sort(Comparator.comparingInt(ChrBaseRegion::start)); + } + } + + public static ExcludedRegions load(final String filename) + { + return new ExcludedRegions(loadRegions(filename)); + } + + // TSV with Chromosome/PosStart/PosEnd columns -> per-chromosome region lists. + private static Map> loadRegions(final String filename) + { + Map> regionsByChr = Maps.newHashMap(); + + try + { + List lines = Files.readAllLines(Paths.get(filename)); + String delim = inferFileDelimiter(filename); + Map fieldIndexMap = createFieldsIndexMap(lines.get(0), delim); + int chrIndex = getChromosomeFieldIndex(fieldIndexMap); + int posStartIndex = getPositionStartFieldIndex(fieldIndexMap); + int posEndIndex = getPositionEndFieldIndex(fieldIndexMap); + + for(int i = 1; i < lines.size(); ++i) + { + String[] values = lines.get(i).split(delim, -1); + String chromosome = values[chrIndex]; + int posStart = Integer.parseInt(values[posStartIndex]); + int posEnd = Integer.parseInt(values[posEndIndex]); + regionsByChr.computeIfAbsent(chromosome, x -> Lists.newArrayList()) + .add(new ChrBaseRegion(chromosome, posStart, posEnd)); + } + } + catch(IOException e) + { + TARS_LOGGER.error("failed to read excluded regions file {}: {}", filename, e.toString()); + return null; + } + + return regionsByChr; + } + + // true if [posStart, posEnd] on chromosome overlaps an excluded region. Called post-lift on genomic coords. + public boolean excludes(final String chromosome, final int posStart, final int posEnd) + { + return overlaps(chromosome, posStart, posEnd); + } + + private boolean overlaps(final String chromosome, final int posStart, final int posEnd) + { + List regions = mRegionsByChr.get(chromosome); + if(regions == null || regions.isEmpty()) + { + return false; + } + + // rightmost region whose start <= posEnd; non-overlapping regions, so it's the only candidate. + int lo = 0; + int hi = regions.size() - 1; + int candidate = -1; + while(lo <= hi) + { + int mid = (lo + hi) >>> 1; + if(regions.get(mid).start() <= posEnd) + { + candidate = mid; + lo = mid + 1; + } + else + { + hi = mid - 1; + } + } + + return candidate >= 0 && regions.get(candidate).end() >= posStart; + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/ExonRegionIndex.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/ExonRegionIndex.java new file mode 100644 index 00000000000..38592ba02f3 --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/ExonRegionIndex.java @@ -0,0 +1,161 @@ +package com.hartwig.hmftools.tars.liftback; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.hartwig.hmftools.common.ensemblcache.EnsemblDataCache; +import com.hartwig.hmftools.common.gene.ExonData; +import com.hartwig.hmftools.common.gene.GeneData; +import com.hartwig.hmftools.common.gene.TranscriptData; +import com.hartwig.hmftools.common.genome.refgenome.RefGenomeVersion; +import com.hartwig.hmftools.common.region.BaseRegion; +import com.hartwig.hmftools.tars.common.ContigEntry; + +// Per-chromosome merged exon spans loaded from the ensembl data cache. Used by LiftBackResolver to +// distinguish a hidden tie where the primary lands in an annotated exon (rescue MAPQ) from a genuinely +// ambiguous tie (keep MAPQ 0). Lookup is O(log N) via binary search over sorted, non-overlapping spans. +// Chromosomes are keyed in the run's ref-genome form (versionedChromosome) so lookups by lifted genomic +// contig match directly. +public final class ExonRegionIndex +{ + private final Map mExonStarts; + private final Map mExonEnds; + + private ExonRegionIndex(final Map starts, final Map ends) + { + mExonStarts = starts; + mExonEnds = ends; + } + + public boolean contains(final String chromosome, final int pos) + { + int[] starts = mExonStarts.get(chromosome); + int[] ends = mExonEnds.get(chromosome); + if(starts == null) + { + return false; + } + + // Binary search for the largest start <= pos; merged intervals guarantee only one candidate. + int lo = 0; + int hi = starts.length - 1; + int idx = -1; + while(lo <= hi) + { + int mid = (lo + hi) >>> 1; + if(starts[mid] <= pos) + { + idx = mid; + lo = mid + 1; + } + else + { + hi = mid - 1; + } + } + return idx >= 0 && pos <= ends[idx]; + } + + public static ExonRegionIndex load(final String ensemblDir, final RefGenomeVersion refGenomeVersion) + { + EnsemblDataCache ensemblDataCache = new EnsemblDataCache(ensemblDir, refGenomeVersion); + ensemblDataCache.setRequiredData(true, false, false, false); + ensemblDataCache.load(false); + return fromCache(ensemblDataCache, refGenomeVersion); + } + + // built from the contig sidecar instead of the ensembl cache: each entry's exonSpans are the genomic exons + // of a multi-exon transcript (the only transcripts that get a tx-contig). Single-exon transcripts are absent, + // but their exons only matter to the discriminator when a read has a tx-contig match -- which only multi-exon + // transcripts produce -- so the derived index is behaviour-equivalent to the ensembl-loaded one. Keyed by the + // sidecar chromosome, the exact form the lift emits. + public static ExonRegionIndex fromContigEntries(final List entries) + { + Map> spansByChromosome = new HashMap<>(); + for(final ContigEntry entry : entries) + { + List spans = spansByChromosome.computeIfAbsent(entry.chromosome(), k -> new ArrayList<>()); + for(final BaseRegion exon : entry.exonSpans()) + { + spans.add(new int[] { exon.start(), exon.end() }); + } + } + return fromSpans(spansByChromosome); + } + + public static ExonRegionIndex fromCache(final EnsemblDataCache ensemblDataCache, final RefGenomeVersion refGenomeVersion) + { + Map> spansByChromosome = new HashMap<>(); + for(final List genes : ensemblDataCache.getChrGeneDataMap().values()) + { + for(final GeneData gene : genes) + { + List transcripts = ensemblDataCache.getTranscripts(gene.GeneId); + if(transcripts == null) + continue; + + String chromosome = refGenomeVersion.versionedChromosome(gene.Chromosome); + List spans = spansByChromosome.computeIfAbsent(chromosome, k -> new ArrayList<>()); + for(final TranscriptData transcript : transcripts) + { + for(final ExonData exon : transcript.exons()) + { + spans.add(new int[] { exon.Start, exon.End }); + } + } + } + } + return fromSpans(spansByChromosome); + } + + private static ExonRegionIndex fromSpans(final Map> spansByChromosome) + { + Map starts = new HashMap<>(); + Map ends = new HashMap<>(); + for(final Map.Entry> entry : spansByChromosome.entrySet()) + { + List merged = mergeIntervals(entry.getValue()); + int[] mergedStarts = new int[merged.size()]; + int[] mergedEnds = new int[merged.size()]; + for(int i = 0; i < merged.size(); ++i) + { + mergedStarts[i] = merged.get(i)[0]; + mergedEnds[i] = merged.get(i)[1]; + } + starts.put(entry.getKey(), mergedStarts); + ends.put(entry.getKey(), mergedEnds); + } + return new ExonRegionIndex(starts, ends); + } + + // Produces sorted, non-overlapping intervals required for safe binary-search lookup. + private static List mergeIntervals(final List intervals) + { + if(intervals.isEmpty()) + { + return intervals; + } + + Collections.sort(intervals, (a, b) -> Integer.compare(a[0], b[0])); + List merged = new ArrayList<>(); + int[] current = new int[] { intervals.get(0)[0], intervals.get(0)[1] }; + for(int i = 1; i < intervals.size(); ++i) + { + int[] next = intervals.get(i); + if(next[0] <= current[1] + 1) + { + current[1] = Math.max(current[1], next[1]); + } + else + { + merged.add(current); + current = new int[] { next[0], next[1] }; + } + } + merged.add(current); + return merged; + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/LiftBackDiscriminator.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/LiftBackDiscriminator.java new file mode 100644 index 00000000000..a671405bd18 --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/LiftBackDiscriminator.java @@ -0,0 +1,445 @@ +package com.hartwig.hmftools.tars.liftback; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +// Pure decision step on a record's full lifted alignment set (self + lifted XA alts). +// categorize() resolves the set to an Outcome (which source wins) plus a DecidingFeature (why); apply() +// mutates Dropped/IsPrimaryChoice for the features that need a swap (e.g. intronless ref alt vs. spliced +// parent gene). No I/O - separate from LiftBackResolver so the decision tree reads as one piece. +public final class LiftBackDiscriminator +{ + private LiftBackDiscriminator() { } + + // Per-alignment evidence emitted into TSV-A for post-hoc "why this outcome" analysis. + public static class Features + { + public DecidingFeature Feature; + public int NumRefAlts; + public int NumTxAlts; + public boolean TxHasNCigar; + public boolean TxSoftClipAtBoundary; + public boolean RefSoftClipped; + public boolean RefFullMatch; + public boolean RefHasNCigar; + } + + public static Features categorize(final List alignments) + { + Features features = new Features(); + + if(alignments.isEmpty()) + { + // Defensive: the primary path always supplies self, so this is unreachable in practice. + features.Feature = null; + return features; + } + + Set loci = new HashSet<>(); + Set distinctCigars = new HashSet<>(); + boolean anyHasN = false; + + for(final LiftedAlignment alignment : alignments) + { + // an alt the overhang gate collapsed to a contiguous alignment is marked Dropped before the + // discriminator runs; it is a fabricated placement, so it must not count toward the features. + if(alignment.Dropped) + { + continue; + } + + loci.add(locusKey(alignment)); + distinctCigars.add(alignment.LiftedCigar); + if(alignment.cigarHasN()) + { + anyHasN = true; + } + + if(alignment.fromTxContig()) + { + ++features.NumTxAlts; + if(alignment.cigarHasRealNJunction()) + { + features.TxHasNCigar = true; + } + if(alignment.SoftClipAtBoundary) + { + features.TxSoftClipAtBoundary = true; + } + } + else + { + ++features.NumRefAlts; + if(alignment.cigarHasSoftClip()) + { + features.RefSoftClipped = true; + } + else + { + features.RefFullMatch = true; + } + if(alignment.cigarHasRealNJunction()) + { + features.RefHasNCigar = true; + } + } + } + + boolean hasRef = features.NumRefAlts > 0; + boolean hasTx = features.NumTxAlts > 0; + + if(loci.size() >= 2) + { + if(hasRef && hasTx) + { + // tx has an annotated junction; ref alts map intronlessly at other loci - usually an intronless + // gene copy, though the rule does not verify that. + if(features.TxHasNCigar && !features.RefHasNCigar) + { + assign(features, DecidingFeature.JUNCTION_OVER_CONTIGUOUS); + } + else + { + assign(features, DecidingFeature.MULTIMAPPER); + } + } + else if(hasTx) + { + assign(features, DecidingFeature.SOLE_TX); + } + else + { + assign(features, DecidingFeature.SOLE_REF); + } + return features; + } + + if(hasRef && !hasTx) + { + assign(features, DecidingFeature.SOLE_REF); + return features; + } + if(hasTx && !hasRef) + { + assign(features, DecidingFeature.SOLE_TX); + return features; + } + + // single locus, both ref and tx - run discriminator + if(distinctCigars.size() == 1 && !anyHasN) + { + assign(features, DecidingFeature.CONCORDANT); + } + else if(features.TxHasNCigar && features.RefSoftClipped) + { + assign(features, DecidingFeature.JUNCTION); + } + else if(features.TxHasNCigar && features.RefFullMatch && !features.RefSoftClipped) + { + // ref full-match across the supposed intron is strong evidence the read is unspliced + // (pre-mRNA / retained intron / DNA contamination) - the tx N-CIGAR is the artifact. + assign(features, DecidingFeature.REF_READS_THROUGH); + } + else if(features.TxSoftClipAtBoundary && features.RefFullMatch) + { + assign(features, DecidingFeature.REF_READS_THROUGH); + } + else + { + assign(features, DecidingFeature.AMBIGUOUS); + } + + return features; + } + + private static void assign(final Features features, final DecidingFeature feature) + { + features.Feature = feature; + } + + public record ApplyResult(LiftedAlignment effectivePrimary, String note) + { + } + + // Deterministic overload: respects bwa's order (no randomization). Used where the caller has no read seed. + public static ApplyResult apply( + final List alignments, final DecidingFeature feature, final LiftedAlignment self) + { + return apply(alignments, feature, self, 0, true); + } + + // Mutates Dropped/IsPrimaryChoice and returns the effective primary and a short diagnostic note. + // When bwa expressed no priority (bwaHasPriority false, ie MAPQ 0) the not-confident outcomes first rank the + // candidates by their recomputed genomic score (set pre-discriminator) and take the clear winner: the + // highest-scoring locus among multi-mappers, or the higher-scoring side of an ambiguous tx-vs-ref call. Only + // when the top candidates tie on score - OR the candidates were never scored (e.g. a split read left for + // supplementary-resolve) - does it fall back to bwa's seed-based pseudo-random pick. bwaHasPriority true leaves + // bwa's order untouched. + public static ApplyResult apply( + final List alignments, final DecidingFeature feature, final LiftedAlignment self, + final int seed, final boolean bwaHasPriority) + { + if(feature == null) + { + return new ApplyResult(self, ""); + } + return switch(feature) + { + case JUNCTION, JUNCTION_OVER_CONTIGUOUS -> promoteTxOverRef(alignments, self); + case REF_READS_THROUGH -> promoteRefOverTx(alignments, self); + case AMBIGUOUS -> bwaHasPriority ? new ApplyResult(self, "") : randomTxVsRef(alignments, self, seed); + case MULTIMAPPER, SOLE_TX, SOLE_REF -> bwaHasPriority ? new ApplyResult(self, "") : randomLocus(alignments, self, seed); + default -> new ApplyResult(self, ""); + }; + } + + // Pick the primary locus among multi-mappers: the highest-scoring locus, with the seeded coin only among loci + // tied at the top. When no candidate was scored (a split read whose scoring is skipped so supplementary-resolve + // can reconstruct it), fall back to bwa's exact behaviour - a seed pick over all loci, first placement there - + // so the merge path is untouched. Every non-dropped placement stays (rides in XA); single-locus sets keep self. + private static ApplyResult randomLocus(final List alignments, final LiftedAlignment self, final int seed) + { + List loci = new ArrayList<>(); + for(final LiftedAlignment la : alignments) + { + if(la.Dropped) + { + continue; + } + String key = locusKey(la); + if(!loci.contains(key)) + { + loci.add(key); + } + } + if(loci.size() < 2) + { + return new ApplyResult(self, ""); + } + + int topScore = Integer.MIN_VALUE; + for(final String locus : loci) + { + topScore = Math.max(topScore, bestScoreAtLocus(alignments, locus)); + } + boolean scored = topScore != Integer.MIN_VALUE; + + List pickable = new ArrayList<>(); + for(final String locus : loci) + { + if(!scored || bestScoreAtLocus(alignments, locus) == topScore) + { + pickable.add(locus); + } + } + + boolean tie = pickable.size() > 1; + String pickedLocus = pickable.get(Math.floorMod(seed, pickable.size())); + if(pickedLocus.equals(locusKey(self))) + { + return new ApplyResult(self, ""); + } + + // unscored: first placement at the locus (bwa behaviour); scored: the best-scoring placement there. + LiftedAlignment winner = null; + for(final LiftedAlignment la : alignments) + { + if(la.Dropped || !locusKey(la).equals(pickedLocus)) + { + continue; + } + if(winner == null || (scored && outranks(la, winner))) + { + winner = la; + } + } + if(winner == null) + { + return new ApplyResult(self, ""); + } + + self.IsPrimaryChoice = false; + winner.IsPrimaryChoice = true; + return new ApplyResult(winner, (scored && !tie) ? "score_locus" : "random_locus"); + } + + private static int bestScoreAtLocus(final List alignments, final String locus) + { + int best = Integer.MIN_VALUE; + for(final LiftedAlignment la : alignments) + { + if(!la.Dropped && locusKey(la).equals(locus)) + { + best = Math.max(best, la.GenomicScore); + } + } + return best; + } + + // Single locus with both a tx and a ref placement and no shape rule to separate them: keep the higher genomic + // score, or a seeded coin when they tie (or were not scored). The loser is a different cigar at the same locus, + // kept in XA (not dropped). + private static ApplyResult randomTxVsRef(final List alignments, final LiftedAlignment self, final int seed) + { + LiftedAlignment tx = bestOfSource(alignments, true); + LiftedAlignment ref = bestOfSource(alignments, false); + if(tx == null || ref == null) + { + return new ApplyResult(self, ""); + } + + LiftedAlignment winner; + String note; + if(bothScored(tx, ref) && tx.GenomicScore != ref.GenomicScore) + { + // a real genomic score separates them, so it is not a true tie: keep the higher-scoring side. + winner = tx.GenomicScore > ref.GenomicScore ? tx : ref; + note = winner == tx ? "score_tx" : "score_ref"; + } + else + { + // genuinely tied (or scores unavailable): fall back to the read-seeded coin so ties stay reproducible. + boolean pickTx = (seed & 1) == 0; + winner = pickTx ? tx : ref; + note = pickTx ? "random_tx" : "random_ref"; + } + for(final LiftedAlignment la : alignments) + { + la.IsPrimaryChoice = la == winner; + } + return new ApplyResult(winner, note); + } + + // Best non-dropped alignment from the tx side (fromTx true) or ref side (fromTx false). + private static LiftedAlignment bestOfSource(final List alignments, final boolean fromTx) + { + LiftedAlignment best = null; + for(final LiftedAlignment la : alignments) + { + if(la.Dropped || la.fromTxContig() != fromTx) + { + continue; + } + if(best == null || outranks(la, best)) + { + best = la; + } + } + return best; + } + + // Higher recomputed genomic score wins; an equal score (or scores not computed) falls back to fewer mismatches. + private static boolean outranks(final LiftedAlignment a, final LiftedAlignment b) + { + if(a.GenomicScore != b.GenomicScore) + { + return a.GenomicScore > b.GenomicScore; + } + return a.NumMismatches < b.NumMismatches; + } + + private static boolean bothScored(final LiftedAlignment a, final LiftedAlignment b) + { + return a.GenomicScore != Integer.MIN_VALUE && b.GenomicScore != Integer.MIN_VALUE; + } + + // Ref-favouring: if self is ref, drop tx alts. If self is tx, swap to the best ref alt. + private static ApplyResult promoteRefOverTx(final List alignments, final LiftedAlignment self) + { + if(!self.fromTxContig()) + { + for(final LiftedAlignment la : alignments) + if(la.fromTxContig()) + { + la.Dropped = true; + } + return new ApplyResult(self, ""); + } + + LiftedAlignment winner = null; + for(final LiftedAlignment la : alignments) + { + if(!la.fromTxContig()) + { + winner = la; + break; + } + } + if(winner == null) + { + return new ApplyResult(self, ""); + } + + self.IsPrimaryChoice = false; + winner.IsPrimaryChoice = true; + for(final LiftedAlignment la : alignments) + { + if(la.fromTxContig() && la != self) + { + la.Dropped = true; + } + } + return new ApplyResult(winner, "swapped_tx_to_ref"); + } + + // Tx-favouring: if self is tx, drop ref alts. If self is ref, swap to the best tx alt (self demoted to XA + // to preserve the contiguous locus). + private static ApplyResult promoteTxOverRef(final List alignments, final LiftedAlignment self) + { + if(self.fromTxContig()) + { + for(final LiftedAlignment la : alignments) + if(!la.fromTxContig()) + { + la.Dropped = true; + } + return new ApplyResult(self, ""); + } + + // Prefer the N-junction tx alt with fewest mismatches; fall back to any tx alt. + LiftedAlignment winner = null; + for(final LiftedAlignment la : alignments) + { + if(la.fromTxContig() && la.cigarHasN()) + { + if(winner == null || la.NumMismatches < winner.NumMismatches) + { + winner = la; + } + } + } + if(winner == null) + { + for(final LiftedAlignment la : alignments) + { + if(la.fromTxContig()) + { + winner = la; + break; + } + } + } + if(winner == null) + { + return new ApplyResult(self, ""); + } + + self.IsPrimaryChoice = false; + winner.IsPrimaryChoice = true; + for(final LiftedAlignment la : alignments) + { + if(!la.fromTxContig() && la != self) + { + la.Dropped = true; + } + } + return new ApplyResult(winner, "swapped_ref_to_tx"); + } + + private static String locusKey(final LiftedAlignment la) + { + return la.LiftedChrom + ":" + la.LiftedPos; + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/LiftBackGroupProcessor.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/LiftBackGroupProcessor.java new file mode 100644 index 00000000000..6eccac2eb48 --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/LiftBackGroupProcessor.java @@ -0,0 +1,920 @@ +package com.hartwig.hmftools.tars.liftback; + +import static com.hartwig.hmftools.tars.liftback.LiftBackRecordOps.markPrimaryUnmapped; +import static com.hartwig.hmftools.tars.liftback.LiftBackRecordOps.refreshNmDropMd; +import static com.hartwig.hmftools.tars.liftback.MateFieldPatcher.patchMateFields; +import static com.hartwig.hmftools.common.bam.SamRecordUtils.SUPPLEMENTARY_ATTRIBUTE; +import static com.hartwig.hmftools.tars.liftback.SaTagRewriter.rewriteSaTag; +import static com.hartwig.hmftools.tars.common.TarsConstants.PRIMARY_AS_UNMAP_THRESHOLD; +import static com.hartwig.hmftools.tars.common.TarsConstants.CONFIDENT_MAPQ; +import static com.hartwig.hmftools.tars.common.TarsConstants.SUPP_AS_DROP_THRESHOLD; +import static com.hartwig.hmftools.tars.common.TarsConfig.TARS_LOGGER; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import com.hartwig.hmftools.common.region.ChrBaseRegion; +import com.hartwig.hmftools.tars.liftback.overhang.OverhangGate; +import com.hartwig.hmftools.tars.liftback.overhang.OverhangGateStatistics; +import com.hartwig.hmftools.tars.liftback.supplementary.SupplementaryResolver; +import com.hartwig.hmftools.tars.liftback.supplementary.RefSequenceSource; +import com.hartwig.hmftools.tars.liftback.supplementary.SupplementaryCandidate; +import com.hartwig.hmftools.tars.liftback.supplementary.SupplementaryResult; +import com.hartwig.hmftools.tars.liftback.supplementary.SupplementaryStatistics; +import com.hartwig.hmftools.tars.liftback.supplementary.SupplementaryRecord; + +import htsjdk.samtools.SAMRecord; +import htsjdk.samtools.TextCigarCodec; +import htsjdk.samtools.util.SequenceUtil; + +// Stateless-per-group transform engine: resolves one contiguous read-name group to genomic coordinates, +// runs the optional supplementary-resolve / overhang-gate passes, patches mate fields against a +// provided LiftedMateInfoCache, and emits each kept record to an EmitSink. The cache is supplied by the +// caller so the standalone TarsApplication can pass its whole-sample pass-1 cache while the REDUX worker +// passes a fresh per-group cache (a complete name-sorted group is self-sufficient -- see +// [[project_redux_migration_scoping]]). Extracted verbatim from TarsApplication so both paths share one +// implementation. +public class LiftBackGroupProcessor +{ + private final LiftBackResolver mResolver; + + // optional supplementary resolver: merges primary + supp across annotated junctions. Null when disabled. + private final SupplementaryResolver mSupplementaryResolver; + + // null when no ref genome is loaded. Runs the overhang gate: iterative terminal collapse + softclip reclaim. + private final OverhangGate mOverhangGate; + + // genomic reference for the post-lift NM recompute. Null when no ref is loaded (supplementary resolve + extend + // both off), in which case NM is cleared rather than recomputed. + private final RefSequenceSource mRefSource; + + // genomic excluded regions (rRNA / contamination), checked post-lift. Null when no regions file configured. + private final ExcludedRegions mExcludedRegions; + + private final LiftBackStats mStats; + + // primaries unmapped because bwa exceeded the XA cap (MAPQ 0 + no XA = maps to too many loci). + private long mOverCapUnmapped; + + // reads dropped because they (primary -> whole fragment, or a supp) lift into an excluded region. + private long mExcludedReads; + + public long overCapUnmapped() { return mOverCapUnmapped; } + + public long excludedReads() { return mExcludedReads; } + + private static final String AS_TAG = "AS"; + + // sink for emitted records: the standalone writes a BAM + TSV, the REDUX worker writes the shared BAM. + public interface EmitSink + { + void emit(SAMRecord record, LiftBackResult result); + } + + public LiftBackGroupProcessor( + final LiftBackResolver resolver, final SupplementaryResolver supplementaryResolver, + final OverhangGate overhangGate, final RefSequenceSource refSource, + final ExcludedRegions excludedRegions, final LiftBackStats stats) + { + mResolver = resolver; + mSupplementaryResolver = supplementaryResolver; + mOverhangGate = overhangGate; + mRefSource = refSource; + mExcludedRegions = excludedRegions; + mStats = stats; + } + + // Processes all records sharing one read name as a group so primary + supps resolve together. This + // lets supplementary resolve see all split-read components and lets /2 hint /1's introns (and vice versa). + public void processNameGroup( + final List group, final LiftedMateInfoCache liftedMateInfoCache, final EmitSink sink) + { + List firstOfPair = new ArrayList<>(); + List secondOfPair = new ArrayList<>(); + for(final SAMRecord record : group) + { + if(!record.getReadPairedFlag() || record.getFirstOfPairFlag()) + { + firstOfPair.add(record); + } + else + { + secondOfPair.add(record); + } + } + + // Two passes per pair so mate /2 can use mate /1's resolved junctions as a hint, and a + // re-decide of mate /1 picks up mate /2's hints when mate /1 wasn't initially resolved. + PassCounterSnapshot beforeM1 = snapshotPassCounters(); + MateDecision m1 = decideMateGroup(firstOfPair, Collections.emptyList()); + PassCounterSnapshot afterM1 = snapshotPassCounters(); + refreshMateInfoCache(firstOfPair, m1, liftedMateInfoCache); + MateDecision m2 = decideMateGroup(secondOfPair, m1.IntroducedIntrons); + refreshMateInfoCache(secondOfPair, m2, liftedMateInfoCache); + + MateDecision m1Final; + if(m1.IntroducedIntrons.isEmpty() && !m2.IntroducedIntrons.isEmpty()) + { + // The provisional m1 decision is discarded and re-made with mate /2's introns. Roll back the engine + // pass-effect counters it bumped (overhang-gate / supplementary resolve / excluded reads) so + // the discarded pass is not double-counted; the re-decision below re-counts. Per-record stats are + // unaffected (recorded once in writeMateGroup on the chosen decision); the emitted BAM is unaffected. + rewindProvisionalCounters(beforeM1, afterM1); + m1Final = decideMateGroup(firstOfPair, m2.IntroducedIntrons); + refreshMateInfoCache(firstOfPair, m1Final, liftedMateInfoCache); + } + else + { + m1Final = m1; + } + + writeMateGroup(firstOfPair, m1Final, liftedMateInfoCache, sink); + writeMateGroup(secondOfPair, m2, liftedMateInfoCache, sink); + } + + // Push the post-supplementary-resolve primary back into the mate cache so the partner mate's MC tag uses + // the merged/extended cigar instead of the pre-resolve one the seed used. + private void refreshMateInfoCache( + final List mateRecords, final MateDecision decision, final LiftedMateInfoCache cache) + { + if(mateRecords.isEmpty() || decision.PrimaryResult == null) + { + return; + } + SAMRecord primary = null; + for(final SAMRecord r : mateRecords) + { + if(!r.getSupplementaryAlignmentFlag()) + { + primary = r; + break; + } + } + if(primary == null) + { + return; + } + LiftedMateInfo refreshed = LiftBackRecordOps.toLiftedMateInfo(primary, decision.PrimaryResult); + boolean firstOfPair = !primary.getReadPairedFlag() || primary.getFirstOfPairFlag(); + cache.recordPrimaryAlignment(primary.getReadName(), firstOfPair, refreshed); + } + + // Snapshot of the pass-effect counters mutated by one decideMateGroup pass, so a discarded provisional mate + // decision can be rolled back rather than double-counted in the summary (see processNameGroup). + private static final class PassCounterSnapshot + { + final long[] Overhang; + final int[] SupplementaryResolve; + final long ExcludedReads; + + PassCounterSnapshot(final long[] overhang, final int[] supplementaryResolve, final long excludedReads) + { + Overhang = overhang; + SupplementaryResolve = supplementaryResolve; + ExcludedReads = excludedReads; + } + } + + private PassCounterSnapshot snapshotPassCounters() + { + return new PassCounterSnapshot( + mOverhangGate != null ? mOverhangGate.statistics().snapshot() : null, + mSupplementaryResolver != null ? mSupplementaryResolver.statistics().snapshot() : null, + mExcludedReads); + } + + // Subtract the (after - before) delta a discarded provisional pass contributed, leaving only the kept counts. + private void rewindProvisionalCounters(final PassCounterSnapshot before, final PassCounterSnapshot after) + { + if(mOverhangGate != null) + { + OverhangGateStatistics overhang = mOverhangGate.statistics(); + long[] current = overhang.snapshot(); + for(int i = 0; i < current.length; ++i) + { + current[i] -= after.Overhang[i] - before.Overhang[i]; + } + overhang.restore(current); + } + if(mSupplementaryResolver != null) + { + SupplementaryStatistics supplementary = mSupplementaryResolver.statistics(); + int[] current = supplementary.snapshot(); + for(int i = 0; i < current.length; ++i) + { + current[i] -= after.SupplementaryResolve[i] - before.SupplementaryResolve[i]; + } + supplementary.restore(current); + } + mExcludedReads -= after.ExcludedReads - before.ExcludedReads; + } + + // Per-mate decision: resolves every record, runs the supplementary resolver, returns the lifted results + + // per-record drop flags + introns introduced by supplementary resolve (for mate hinting). Does not emit. + private static final class MateDecision + { + LiftBackResult[] Resolved; + boolean[] DroppedBySupplementary; + LiftBackResult PrimaryResult; + List IntroducedIntrons; + int PrimaryIndex; + boolean PrimaryPostProcessed; + + MateDecision(final LiftBackResult[] resolved, final boolean[] droppedBySupplementary, + final LiftBackResult primaryResult, final List introducedIntrons, + final int primaryIndex, final boolean primaryPostProcessed) + { + Resolved = resolved; + DroppedBySupplementary = droppedBySupplementary; + PrimaryResult = primaryResult; + IntroducedIntrons = introducedIntrons != null ? introducedIntrons : Collections.emptyList(); + PrimaryIndex = primaryIndex; + PrimaryPostProcessed = primaryPostProcessed; + } + + static MateDecision empty() + { + return new MateDecision(new LiftBackResult[0], null, null, Collections.emptyList(), -1, false); + } + } + + private MateDecision decideMateGroup(final List records, final List mateHintIntrons) + { + if(records.isEmpty()) + { + return MateDecision.empty(); + } + + SAMRecord primary = null; + for(final SAMRecord record : records) + { + if(record.getSupplementaryAlignmentFlag()) + continue; + if(primary == null) + { + primary = record; + } + } + + // a valid BAM always has a primary in the mate group, so resolve it directly. Candidate cigars are peeled + // by the overhang gate before the discriminator runs - see reconcileAlignmentsToGenome. + LiftBackResult primaryResult = mResolver.resolve(primary, this::reconcileAlignmentsToGenome); + + LiftBackResult[] resolved = new LiftBackResult[records.size()]; + for(int i = 0; i < records.size(); i++) + { + SAMRecord record = records.get(i); + resolved[i] = (record == primary && primaryResult != null) + ? primaryResult : mResolver.resolve(record, this::reconcileAlignmentsToGenome); + } + + int primaryIdx = primary != null ? indexOfPrimary(records, primary) : -1; + + List introduced = new ArrayList<>(); + boolean[] droppedBySupplementary = applySupplementaryResolve(records, resolved, primary, mateHintIntrons, introduced); + + // Finalize the chosen primary: peel it, then reclaim its terminal softclip (tx-match only). Not a redundant + // repeat of the per-candidate peel in reconcileAlignmentsToGenome - it does real work in two cases: + // 1. supplementary resolve merged a primary+supp into a fresh M N M cigar that never existed at candidate + // time, so it can carry its own fabricated terminal micro-junction; and + // 2. the standalone softclip reclaim is deliberately deferred to here (post-resolve) so supplementary + // resolve saw the original clip - this is the chosen primary's first and only reclaim. + // A clean primary is left unchanged by each step. + reconcileChosenPrimary(records, resolved, primary, droppedBySupplementary); + + // post-lift exclusion: if the final primary placement lands in an excluded (rRNA / contamination) region, + // unmap it REDUX-style - flip the result to UNMAPPED so the mate is coordinated via the cache (willBeUnmapped) + // and the record is unmapped at apply time. Lifted genomic coords are the only space tx-contig reads can be + // tested against the genomic region list, so this is post-lift, not the old pre-lift fragment pre-filter. + if(primaryIdx >= 0 && liftsIntoExcludedRegion(resolved[primaryIdx])) + { + resolved[primaryIdx] = LiftBackResult.unmapped(LiftBackResult.RecordRole.PRIMARY, "excluded_region_unmapped"); + ++mExcludedReads; + } + + // supplementary resolve / the overhang gate each replace resolved[primaryIdx] with a new result object when + // they improve the primary. A changed reference means liftback post-processed it, so its stale bwa + // AS must not be used to AS-filter it (see PRIMARY_AS_UNMAP_THRESHOLD). + boolean primaryPostProcessed = primaryIdx >= 0 && resolved[primaryIdx] != primaryResult; + + return new MateDecision(resolved, droppedBySupplementary, + primaryIdx >= 0 ? resolved[primaryIdx] : primaryResult, + introduced, primaryIdx, primaryPostProcessed); + } + + private static int indexOfPrimary(final List records, final SAMRecord primary) + { + for(int i = 0; i < records.size(); ++i) + if(records.get(i) == primary) + { + return i; + } + return -1; + } + + private void writeMateGroup( + final List records, final MateDecision decision, + final LiftedMateInfoCache liftedMateInfoCache, final EmitSink sink) + { + if(records.isEmpty()) + { + return; + } + + LiftBackResult[] resolved = decision.Resolved; + boolean[] droppedBySupplementary = decision.DroppedBySupplementary; + SAMRecord primary = null; + for(SAMRecord r : records) + if(!r.getSupplementaryAlignmentFlag()) + { + primary = r; + break; + } + + // A supplementary whose primary ends up unmapped (e.g. the primary lifted into an excluded region) is + // orphaned: there is no emitted primary for its SA to reference, so REDUX FragmentCoords would read a + // dangling primary locus off it. Drop the whole group's supps in that case. + boolean primaryUnmapped = decision.PrimaryResult == null || LiftBackRecordOps.willBeUnmapped(decision.PrimaryResult); + + // Dedup supplementaries that lift to the same (chrom, pos, cigar, strand) -- bwa can emit the + // same junction across multiple transcript contigs and they collapse after liftback. + Set emittedSuppKeys = new HashSet<>(); + boolean[] willEmit = new boolean[records.size()]; + for(int i = 0; i < records.size(); i++) + { + SAMRecord record = records.get(i); + LiftBackResult result = resolved[i]; + boolean drop = droppedBySupplementary != null && droppedBySupplementary[i]; + if(!drop && record.getSupplementaryAlignmentFlag() && primaryUnmapped) + { + drop = true; + mStats.recordOrphanSuppDropped(); + } + if(!drop && record != primary && record.getSupplementaryAlignmentFlag()) + { + String key = dedupKey(result, record); + if(!emittedSuppKeys.add(key)) + { + drop = true; + } + } + // Drop supps the supplementary-resolve pass left behind that exist only because bwa-mem2 was run with + // -T 19 below its default of 30 (see SUPP_AS_DROP_THRESHOLD). Only applies when supplementary resolve + // ran, because a configuration without it might want to retain these supps for other reasons. + if(!drop && mSupplementaryResolver != null && record.getSupplementaryAlignmentFlag() + && !record.getReadUnmappedFlag()) + { + Integer alignmentScore = record.getIntegerAttribute(AS_TAG); + if(alignmentScore != null && alignmentScore < SUPP_AS_DROP_THRESHOLD) + { + drop = true; + mStats.recordLowAsSuppDropped(); + } + } + // a supp lifting into an excluded region is dropped (a supp can't be unmapped); its SA entry is + // removed from the primary below so the primary doesn't reference a supp that isn't emitted. + if(!drop && record.getSupplementaryAlignmentFlag() && liftsIntoExcludedRegion(result)) + { + drop = true; + ++mExcludedReads; + } + willEmit[i] = !drop; + } + + // SA entries of dropped supps removed from the primary's SA so it never references a missing supp. + Set droppedSuppSaKeys = new HashSet<>(); + for(int i = 0; i < records.size(); i++) + { + if(willEmit[i]) + continue; + SAMRecord record = records.get(i); + LiftBackResult result = resolved[i]; + if(record == primary || !record.getSupplementaryAlignmentFlag() || result == null + || result.finalChrom() == null || result.finalCigar() == null + || result.finalCigar().equals(SAMRecord.NO_ALIGNMENT_CIGAR)) + continue; + droppedSuppSaKeys.add(suppSaKey(result)); + } + + // NH = the number of distinct genomic loci the read lifts back to, taken from the resolver's + // chrom:pos-keyed locus count. Counting emitted records would inflate NH by tx-contig + // representation multiplicity (one junction repeated across many transcript contigs all lifting + // to the same locus). + int nh = decision.PrimaryResult != null ? Math.max(decision.PrimaryResult.numLoci(), 1) : 1; + + // When the discriminator moved the primary off bwa's placement (a ref->tx swap or a random-locus pick), each + // surviving supplementary's own bwa SA still references bwa's PRE-move primary locus, which was never emitted. + // Rebuild those supp SAs to reference the emitted primary so REDUX FragmentCoords reads the right primary coords. + boolean primarySwapped = !primaryUnmapped && decision.PrimaryResult != null && decision.PrimaryResult.swapped(); + + for(int i = 0; i < records.size(); i++) + { + SAMRecord record = records.get(i); + LiftBackResult result = resolved[i]; + if(!willEmit[i]) + { + mStats.record(record, result); + continue; + } + boolean primaryPostProcessed = i == decision.PrimaryIndex && decision.PrimaryPostProcessed; + String rebuiltSuppSa = (primarySwapped && record != primary && record.getSupplementaryAlignmentFlag()) + ? buildSwappedSuppSa(records, resolved, willEmit, decision.PrimaryResult, i) : null; + applyAndWriteRecord(record, result, nh, primaryPostProcessed, droppedSuppSaKeys, liftedMateInfoCache, sink, rebuiltSuppSa); + } + } + + // SA for a supplementary whose primary was moved: the emitted primary entry followed by any sibling emitted supps. + private static String buildSwappedSuppSa( + final List records, final LiftBackResult[] resolved, final boolean[] willEmit, + final LiftBackResult primaryResult, final int suppIndex) + { + if(primaryResult.finalChrom() == null || primaryResult.finalCigar() == null) + { + return null; + } + StringBuilder sb = new StringBuilder(saEntry(primaryResult)); + for(int i = 0; i < records.size(); ++i) + { + if(i == suppIndex || !willEmit[i] || !records.get(i).getSupplementaryAlignmentFlag()) + { + continue; + } + LiftBackResult res = resolved[i]; + if(res == null || res.finalChrom() == null || res.finalCigar() == null + || res.finalCigar().equals(SAMRecord.NO_ALIGNMENT_CIGAR)) + { + continue; + } + sb.append(saEntry(res)); + } + return sb.length() > 0 ? sb.toString() : null; + } + + // one SA entry (chrom,pos,strand,cigar,mapq,NM;) for a lifted alignment; NM is left 0 (informational -- REDUX keys on coords). + private static String saEntry(final LiftBackResult result) + { + return result.finalChrom() + ',' + result.finalPos() + ',' + (result.negativeStrand() ? '-' : '+') + + ',' + result.finalCigar() + ',' + result.updatedMapq() + ",0;"; + } + + // builds a SupplementaryCandidate from the post-lift primary + its supplementary records and applies the + // merge if accepted. Returns a parallel array marking which records were absorbed by the merge and + // should be dropped from emission. Returns null when supplementary resolve is disabled / no-op. + // introducedIntronsOut is appended-to (caller-provided list) so the partner mate can use them as hints in + // the second pass. + private boolean[] applySupplementaryResolve( + final List records, final LiftBackResult[] resolved, final SAMRecord primary, + final List mateHintIntrons, final List introducedIntronsOut) + { + if(mSupplementaryResolver == null || primary == null || primary.getReadUnmappedFlag()) + { + return null; + } + + int primaryIdx = -1; + List suppIndices = new ArrayList<>(); + for(int i = 0; i < records.size(); i++) + { + SAMRecord r = records.get(i); + if(r == primary) + { + primaryIdx = i; + } + else if(r.getSupplementaryAlignmentFlag() && !r.getReadUnmappedFlag()) + { + suppIndices.add(i); + } + } + if(primaryIdx < 0) + { + return null; + } + + LiftBackResult primaryRes = resolved[primaryIdx]; + if(primaryRes == null || primaryRes.finalCigar() == null + || primaryRes.finalCigar().equals(SAMRecord.NO_ALIGNMENT_CIGAR)) + return null; + + List suppDtos = new ArrayList<>(suppIndices.size()); + for(int i = 0; i < suppIndices.size(); i++) + { + int idx = suppIndices.get(i); + SAMRecord supp = records.get(idx); + LiftBackResult suppRes = resolved[idx]; + if(suppRes == null || suppRes.finalCigar() == null + || suppRes.finalCigar().equals(SAMRecord.NO_ALIGNMENT_CIGAR)) + continue; + suppDtos.add(new SupplementaryRecord( + i, suppRes.finalChrom(), !supp.getReadNegativeStrandFlag(), + suppRes.finalPos(), suppRes.finalCigar(), supp.getMappingQuality())); + } + + SupplementaryCandidate candidate = new SupplementaryCandidate( + primaryRes.finalChrom(), !primary.getReadNegativeStrandFlag(), primary.getReadLength(), + primaryRes.finalPos(), primaryRes.finalCigar(), primary.getMappingQuality(), suppDtos, + primary.getReadBases(), mateHintIntrons); + + SupplementaryResult result = mSupplementaryResolver.resolve(candidate); + if(!result.merged()) + { + return null; + } + + // rewrite the primary's LiftBackResult with the merged (now-spliced) cigar + start; the start may + // shift for a left-extend. Merged MAPQ is the max of the primary and every merged supp, then bumped to + // 60 only when the primary+supplementary pair maps to a single locus: the merge already requires exactly + // one supplementary within reach on that side, so the pair's uniqueness is the primary lifting to one + // locus (no competing XA alt). A pair that still maps to more than one locus keeps its bwa MAPQ. + int mergedMapq = primaryRes.updatedMapq(); + for(Integer dtoIdx : result.droppedSupplementaryIndices()) + { + mergedMapq = Math.max(mergedMapq, suppDtos.get(dtoIdx).mapq()); + } + if(primaryRes.numLoci() == 1) + { + mergedMapq = CONFIDENT_MAPQ; + } + resolved[primaryIdx] = primaryRes.withRevisedCigar( + result.mergedStart(), result.mergedCigar(), true, + mergedMapq, "supplementary-resolved"); + TARS_LOGGER.debug2("supplementary-resolved {}: {}:{} {} -> {} (depth {})", + primary.getReadName(), primaryRes.finalChrom(), primaryRes.finalPos(), + primaryRes.finalCigar(), result.mergedCigar(), result.chainDepth()); + + if(introducedIntronsOut != null) + { + introducedIntronsOut.addAll(result.introducedIntrons()); + } + + boolean[] dropped = new boolean[records.size()]; + for(Integer dtoIdx : result.droppedSupplementaryIndices()) + { + dropped[suppIndices.get(dtoIdx)] = true; + } + return dropped; + } + + // Peels each candidate's terminal junction overhangs against the genome before the discriminator, so its + // RefSoftClipped / RefFullMatch / TxHasNCigar inputs are measured facts, not raw-bwa guesses. Read bases are + // taken in each candidate's own orientation; self and same-strand alts use the record bases as-is, + // opposite-strand alts the reverse complement. Only the peel runs here (no standalone reclaim): self's softclip + // is left intact for supplementary resolve (a primary+supp merge needs the clip), and the chosen primary is + // reclaimed post-resolve in reconcileChosenPrimary. An XA alt the peel collapses to a purely contiguous alignment is a + // fabricated placement and is dropped from the tag + the locus count; self is never dropped, only collapsed. + private void reconcileAlignmentsToGenome(final List alignments, final SAMRecord record) + { + if(mOverhangGate == null || !mOverhangGate.enabled() || alignments.isEmpty()) + { + return; + } + + byte[] forwardBases = record.getReadBases(); + if(forwardBases == null || forwardBases.length == 0) + { + return; + } + + LiftedAlignment self = alignments.get(0); + boolean recordForward = !record.getReadNegativeStrandFlag(); + + // Fast path for the dominant single-candidate read: peel self directly and skip the per-placement dedup map. + // Self uses the record's own orientation (forward bases) and is never dropped. + if(alignments.size() == 1) + { + if(self.LiftedCigar != null) + { + OverhangGate.Result peeled = mOverhangGate.peel( + self.LiftedChrom, self.LiftedPos, self.LiftedCigar, forwardBases); + if(peeled.pos() != self.LiftedPos || !peeled.cigar().equals(self.LiftedCigar)) + { + alignments.set(0, self.withLiftedCigar(peeled.pos(), peeled.cigar())); + } + } + return; + } + + byte[] reverseBases = null; + + // A read with a supplementary is a split read: supplementary-resolve reconstructs its true spliced alignment + // more faithfully than any single lifted half. Scoring the pre-merge halves would let the discriminator prefer + // a clean contiguous alt over the short-anchor placement supp-resolve needs, so leave GenomicScore unset here + // (the discriminator then keeps its seeded-random tie-break) and let the merge run. + boolean recordHasSupplementary = record.getStringAttribute(SUPPLEMENTARY_ATTRIBUTE) != null; + + // Many candidates lift to an identical genomic placement (packed isoform contigs of one gene all collapsing + // to one locus). peel() is deterministic in (chrom, pos, cigar, strand), so each distinct placement is + // peeled once and the result reused for every duplicate. + Map placementsByKey = new HashMap<>(); + + for(int i = 0; i < alignments.size(); ++i) + { + LiftedAlignment alignment = alignments.get(i); + if(alignment.LiftedCigar == null) + { + continue; + } + + // read bases in this candidate's orientation (reverse-complemented for an opposite-strand alt); used + // both to peel the terminal overhangs and to recompute the genomic score below. + byte[] bases; + if(alignment.ForwardStrand == recordForward) + { + bases = forwardBases; + } + else + { + if(reverseBases == null) + { + reverseBases = Arrays.copyOf(forwardBases, forwardBases.length); + SequenceUtil.reverseComplement(reverseBases); + } + bases = reverseBases; + } + + String key = alignment.LiftedChrom + ':' + alignment.LiftedPos + ':' + alignment.LiftedCigar + + (alignment.ForwardStrand ? '+' : '-'); + OverhangGate.Result peeled = placementsByKey.get(key); + if(peeled == null) + { + peeled = mOverhangGate.peel(alignment.LiftedChrom, alignment.LiftedPos, alignment.LiftedCigar, bases); + placementsByKey.put(key, peeled); + } + + // An XA alt whose junction(s) fully collapsed is a fabricated placement: drop it from XA + the locus + // count. Self keeps its collapsed cigar (it stays the record's placement). + if(alignment != self && peeled.dropped()) + { + alignment.Dropped = true; + mOverhangGate.statistics().countAltDropped(); + continue; + } + + if(peeled.pos() != alignment.LiftedPos || !peeled.cigar().equals(alignment.LiftedCigar)) + { + alignment = alignment.withLiftedCigar(peeled.pos(), peeled.cigar()); + alignments.set(i, alignment); + } + + // recompute the genomic score on the final placement so Step 2 can rank candidates before any + // seeded-random tie-break (bwa's AlignmentScore is not comparable across tx and ref candidates). + // Skipped for split reads (see recordHasSupplementary above) so their supplementary-resolve merge stands. + if(!recordHasSupplementary) + { + alignment.GenomicScore = AlignmentScorer.score( + mRefSource, alignment.LiftedChrom, alignment.LiftedPos, alignment.LiftedCigar, bases); + } + } + } + + // True when the chosen primary's placement was lifted from a transcript contig; gates the standalone softclip + // reclaim so a genomic/ref over-clip is left as bwa placed it. + private static boolean isTxMatchPrimary(final LiftBackResult primaryRes) + { + for(final LiftedAlignment alignment : primaryRes.liftedAlignments()) + { + if(alignment.IsPrimaryChoice) + { + return alignment.fromTxContig(); + } + } + return false; + } + + // Finalize the chosen primary: peel its terminal junction overhangs (catches one merge fabricated in a fresh + // M N M cigar), reclaim a surviving terminal softclip when the primary is a tx-match (deferred here so + // supplementary resolve saw the original clip). Runs on whatever primary the discriminator or supplementary + // resolve chose. A clean primary is unchanged by each step. + private void reconcileChosenPrimary( + final List records, final LiftBackResult[] resolved, final SAMRecord primary, + final boolean[] droppedBySupplementary) + { + if(mOverhangGate == null) + { + return; + } + if(primary == null || primary.getReadUnmappedFlag()) + { + return; + } + + int primaryIdx = indexOfPrimary(records, primary); + if(primaryIdx < 0 || (droppedBySupplementary != null && droppedBySupplementary[primaryIdx])) + { + return; + } + + LiftBackResult primaryRes = resolved[primaryIdx]; + if(primaryRes == null || primaryRes.finalCigar() == null + || primaryRes.finalCigar().equals(SAMRecord.NO_ALIGNMENT_CIGAR)) + return; + + // The discriminator may have swapped the primary to an opposite-strand alt. The gate walks the read bases + // against the genome in the placement's orientation, so reverse-complement them when the resolved strand + // differs from the record's own (matches the per-candidate handling in reconcileAlignmentsToGenome). + byte[] readBases = primary.getReadBases(); + if(readBases != null && primaryRes.negativeStrand() != primary.getReadNegativeStrandFlag()) + { + readBases = Arrays.copyOf(readBases, readBases.length); + SequenceUtil.reverseComplement(readBases); + } + + // Peel the chosen primary (catches a fabricated terminal micro-junction merge created in a fresh M N M), + // then reclaim a surviving terminal softclip - but only for a tx-match, and only here (post-resolve) so + // supplementary resolve saw the original clip and a genomic over-clip is left untouched. A clean primary is unchanged. + if(mOverhangGate != null && mOverhangGate.enabled()) + { + OverhangGate.Result peeled = mOverhangGate.peel( + primaryRes.finalChrom(), primaryRes.finalPos(), primaryRes.finalCigar(), readBases); + int newPos = peeled.pos(); + String newCigar = peeled.cigar(); + + if(isTxMatchPrimary(primaryRes)) + { + OverhangGate.Result reclaimed = mOverhangGate.reclaimTerminalSoftClip( + primaryRes.finalChrom(), newPos, newCigar, readBases); + newPos = reclaimed.pos(); + newCigar = reclaimed.cigar(); + } + + if(newPos != primaryRes.finalPos() || !newCigar.equals(primaryRes.finalCigar())) + { + TARS_LOGGER.debug2("overhang-gate primary {}: {} -> {}", + primary.getReadName(), primaryRes.finalCigar(), newCigar); + primaryRes = primaryRes.withRevisedCigar( + newPos, newCigar, newCigar.indexOf('N') >= 0, + primaryRes.updatedMapq(), "overhang-gated"); + resolved[primaryIdx] = primaryRes; + } + + // Reclaim terminal softclips on tx-match XA alts as well. Supplementaries only ever merge into the + // primary, so an alt's clip is never a splice the resolver still needs and is safe to consume here. + // Genomic alts keep bwa's clip. Mutates the shared liftedAlignments list that buildLiftedXa emits from. + reclaimTxMatchAlts(primaryRes.liftedAlignments(), primary); + } + } + + // Reclaim a surviving terminal softclip on each tx-match XA alt, mirroring the chosen primary's reclaim above. + // Genomic alts (bwa's clip is a genuine mismatch call) and the primary are skipped. Read bases are taken in each + // alt's placement orientation, matching the per-candidate handling in reconcileAlignmentsToGenome. + private void reclaimTxMatchAlts(final List alignments, final SAMRecord primary) + { + byte[] forwardBases = primary.getReadBases(); + if(forwardBases == null || forwardBases.length == 0) + { + return; + } + + boolean recordForward = !primary.getReadNegativeStrandFlag(); + byte[] reverseBases = null; + + for(int i = 0; i < alignments.size(); ++i) + { + LiftedAlignment alt = alignments.get(i); + if(alt.IsPrimaryChoice || alt.Dropped || !alt.fromTxContig()) + { + continue; + } + if(alt.LiftedCigar == null || alt.LiftedCigar.indexOf('S') < 0) + { + continue; + } + + byte[] bases; + if(alt.ForwardStrand == recordForward) + { + bases = forwardBases; + } + else + { + if(reverseBases == null) + { + reverseBases = Arrays.copyOf(forwardBases, forwardBases.length); + SequenceUtil.reverseComplement(reverseBases); + } + bases = reverseBases; + } + + OverhangGate.Result reclaimed = mOverhangGate.reclaimTerminalSoftClip( + alt.LiftedChrom, alt.LiftedPos, alt.LiftedCigar, bases); + if(reclaimed.pos() != alt.LiftedPos || !reclaimed.cigar().equals(alt.LiftedCigar)) + { + alignments.set(i, alt.withLiftedCigar(reclaimed.pos(), reclaimed.cigar())); + } + } + } + + // Dedup key for supplementaries: lifted (chrom, pos, cigar) + lifted strand. Opposite-strand + // placements at the same coords/cigar are kept as distinct records. + private static String dedupKey(final LiftBackResult result, final SAMRecord record) + { + return result.finalChrom() + ":" + result.finalPos() + ":" + result.finalCigar() + + ":" + (result.negativeStrand() ? '-' : '+'); + } + + // true if the result's lifted genomic span overlaps an excluded region. False for unmapped / unlifted results. + private boolean liftsIntoExcludedRegion(final LiftBackResult result) + { + if(mExcludedRegions == null || result == null) + { + return false; + } + String cigar = result.finalCigar(); + if(result.finalChrom() == null || cigar == null || cigar.equals(SAMRecord.NO_ALIGNMENT_CIGAR)) + { + return false; + } + int end = result.finalPos() + TextCigarCodec.decode(cigar).getReferenceLength() - 1; + return mExcludedRegions.excludes(result.finalChrom(), result.finalPos(), end); + } + + // SA entry key (chrom:pos:strand:cigar) of a supplementary's lifted placement, matching SaTagRewriter's + // entry key so a dropped supp's entry can be removed from the primary's SA tag. Clips are normalised H->S: + // the supp RECORD is hard-clipped (e.g. 19M247H) but the primary's SA tag lists it soft-clipped (19M247S), + // so the keys would otherwise never match and the dropped supp's SA entry would dangle. + private static String suppSaKey(final LiftBackResult result) + { + return result.finalChrom() + ":" + result.finalPos() + + ":" + (result.negativeStrand() ? '-' : '+') + ":" + result.finalCigar().replace('H', 'S'); + } + + private void applyAndWriteRecord( + final SAMRecord record, final LiftBackResult result, final int nh, final boolean primaryPostProcessed, + final Set droppedSuppSaKeys, final LiftedMateInfoCache liftedMateInfoCache, final EmitSink sink, + final String rebuiltSuppSa) + { + mStats.record(record, result); + + LiftBackRecordOps.applyResultToRecord(record, result, liftedMateInfoCache); + + // A primary flipped to UNMAPPED post-lift (e.g. excluded region) still carries its mapped coords - + // applyResultToRecord's UNMAPPED case is a no-op for input-unmapped reads, so unmap it explicitly. + if(result.recordState() == RecordState.UNMAPPED && !record.getReadUnmappedFlag()) + { + markPrimaryUnmapped(record); + } + + if(record.getReadUnmappedFlag()) + record.setReadNegativeStrandFlag(false); + + // rebuiltSuppSa is set only for a supplementary whose primary the discriminator moved: use the primary-referencing + // SA rebuilt from the emitted group. Otherwise translate this record's own bwa SA to genomic coords as usual. + String rewrittenSa = rebuiltSuppSa != null + ? rebuiltSuppSa + : rewriteSaTag(record.getStringAttribute(SUPPLEMENTARY_ATTRIBUTE), mResolver, droppedSuppSaKeys); + + // A record that stays supplementary must carry an SA tag -- REDUX dedup (FragmentCoords.fromRead) + // reads the primary's coords from it. If every SA entry failed to lift the supplementary is orphaned + // in genomic space and cannot be represented, so drop it rather than emit a supp with a null SA. + if(rewrittenSa == null && record.getSupplementaryAlignmentFlag()) + { + mStats.recordOrphanSuppDropped(); + return; + } + + record.setAttribute(SUPPLEMENTARY_ATTRIBUTE, rewrittenSa); + patchMateFields(record, liftedMateInfoCache); + + if(result.recordState() != RecordState.UNMAPPED) + { + record.setAttribute("NH", nh); + } + + // Over bwa's XA reporting cap: a primary that bwa emitted MAPQ 0 with no XA maps to too many loci to + // place (the alt list was suppressed, not truncated). Unmap it. Keyed on the input state because the + // missing XA makes the resolver see a single locus, which would otherwise bump MAPQ to 60. + if(LiftBackRecordOps.exceedsMappingCap(result) + && !record.getReadUnmappedFlag() + && !record.getSupplementaryAlignmentFlag()) + { + markPrimaryUnmapped(record); + ++mOverCapUnmapped; + TARS_LOGGER.debug2("over-cap unmap {}: inputMapq=0, no XA (past bwa XA cap)", record.getReadName()); + } + + // residual short-anchor primary: bwa scored it below the default -T 30 floor and liftback couldn't + // improve it (not resolved / extended / collapsed). Unmap it so the noise alignment doesn't pollute + // the locus. Gated on supplementary resolve running, matching the supp drop's -T 19 rationale. + if(mSupplementaryResolver != null && !primaryPostProcessed + && !record.getReadUnmappedFlag() + && !record.getSupplementaryAlignmentFlag()) + { + Integer alignmentScore = record.getIntegerAttribute(AS_TAG); + if(alignmentScore != null && alignmentScore < PRIMARY_AS_UNMAP_THRESHOLD) + { + markPrimaryUnmapped(record); + mStats.recordLowAsPrimaryUnmapped(); + TARS_LOGGER.debug2("AS-floor unmap {}: AS={} < {}", record.getReadName(), alignmentScore, PRIMARY_AS_UNMAP_THRESHOLD); + } + } + + refreshNmDropMd(record, mRefSource); + + sink.emit(record, result); + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/LiftBackRecordOps.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/LiftBackRecordOps.java new file mode 100644 index 00000000000..c158961fb7b --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/LiftBackRecordOps.java @@ -0,0 +1,315 @@ +package com.hartwig.hmftools.tars.liftback; + +import static com.hartwig.hmftools.common.bam.SamRecordUtils.SUPPLEMENTARY_ATTRIBUTE; +import static com.hartwig.hmftools.common.bam.SamRecordUtils.XA_ATTRIBUTE; +import static com.hartwig.hmftools.common.region.BaseRegion.positionsOverlap; + +import java.util.stream.Collectors; + +import com.hartwig.hmftools.tars.liftback.supplementary.RefSequenceSource; + +import htsjdk.samtools.Cigar; +import htsjdk.samtools.CigarElement; +import htsjdk.samtools.SAMRecord; +import htsjdk.samtools.TextCigarCodec; +import htsjdk.samtools.util.SequenceUtil; + +// Pure per-record mutation primitives shared by the standalone TarsApplication main and the REDUX +// LiftBackGroupProcessor. No group/orchestration state -- each method maps a resolved LiftBackResult onto +// a single SAMRecord (or derives the mate-cache view of it). Extracted from TarsApplication so the +// concurrent REDUX path and the standalone path share one implementation. +public final class LiftBackRecordOps +{ + private LiftBackRecordOps() { } + + // Supplementaries whose own lift failed are mirrored onto their primary's lifted coords (keeping + // the 0x800 flag) rather than marked unmapped -- htsjdk's validator rejects 0x4+0x800. + public static void applyResultToRecord( + final SAMRecord record, final LiftBackResult result, final LiftedMateInfoCache liftedMateInfoCache) + { + switch(result.recordState()) + { + case UNMAPPED: + return; + + case LIFT_FAILED: + if(record.isSecondaryOrSupplementary() && mirrorOwnPrimaryOntoFailedSupp(record, liftedMateInfoCache)) + { + return; + } + + record.setReadUnmappedFlag(true); + record.setReadNegativeStrandFlag(false); + record.setReferenceName(SAMRecord.NO_ALIGNMENT_REFERENCE_NAME); + record.setAlignmentStart(SAMRecord.NO_ALIGNMENT_START); + record.setCigarString(SAMRecord.NO_ALIGNMENT_CIGAR); + record.setMappingQuality(0); + record.setAttribute(XA_ATTRIBUTE, null); + record.setAttribute(SUPPLEMENTARY_ATTRIBUTE, null); + return; + + default: + final Cigar liftedCigar = new Cigar(ContigTranslator.mergeAdjacentSameOp( + ContigTranslator.dropZeroLength(TextCigarCodec.decode(result.finalCigar()).getCigarElements()))); + + // A discriminator swap can move the primary onto an opposite-strand placement (bwa's contiguous + // alignment sits on one strand while the winning spliced tx placement is on the other). SEQ and base + // qualities are stored in the input record's orientation; when the resolved strand flips, reverse- + // complement the bases and reverse the qualities so SEQ stays on the genomic forward strand and + // matches the lifted position/cigar. Without this the read is emitted mis-oriented and the recomputed + // NM (refreshNmDropMd) is measured against the wrong-strand bases -- inflating it to ~read length. + if(result.negativeStrand() != record.getReadNegativeStrandFlag()) + { + byte[] bases = record.getReadBases(); + if(bases != null && bases.length > 0) + { + byte[] flipped = bases.clone(); + SequenceUtil.reverseComplement(flipped); + record.setReadBases(flipped); + } + byte[] quals = record.getBaseQualities(); + if(quals != null && quals.length > 0) + { + byte[] reversed = quals.clone(); + for(int i = 0, j = reversed.length - 1; i < j; ++i, --j) + { + byte tmp = reversed[i]; + reversed[i] = reversed[j]; + reversed[j] = tmp; + } + record.setBaseQualities(reversed); + } + } + + record.setReferenceName(result.finalChrom()); + record.setAlignmentStart(result.finalPos()); + record.setCigar(liftedCigar); + record.setReadNegativeStrandFlag(result.negativeStrand()); + record.setMappingQuality(result.updatedMapq()); + record.setAttribute(XA_ATTRIBUTE, buildLiftedXa(result)); + // XS:A:+/- on spliced records: downstream RNA tools (Isofox) rely on + // transcript strand for stranded junction interpretation. bwa-mem2 emits XS:i: as + // the sub-optimal alignment score on the same tag name -- clear first so the SAM + // tag-type bookkeeping isn't ambiguous. Only set XS:A when the lifted cigar has + // an N AND we know the transcript strand (i.e. the primary came off a tx contig). + // Ref-only N-cigars from supplementary-resolve/tail-extend don't have a strand threaded yet -- + // they ship without XS rather than risk a wrong call. + record.setAttribute("XS", null); + if(result.hasNCigar() && result.transcriptStrand() != 0) + { + record.setAttribute("XS", result.transcriptStrand() > 0 ? Character.valueOf('+') : Character.valueOf('-')); + } + } + } + + private static boolean mirrorOwnPrimaryOntoFailedSupp( + final SAMRecord record, final LiftedMateInfoCache liftedMateInfoCache) + { + if(!record.getReadPairedFlag()) + { + return false; + } + + LiftedMateInfo ownPrimary = liftedMateInfoCache.getOwnPrimary(record.getReadName(), record.getFirstOfPairFlag()); + if(ownPrimary == null || ownPrimary.unmapped()) + { + return false; + } + + record.setReferenceName(ownPrimary.chromosome()); + record.setAlignmentStart(ownPrimary.alignmentStart()); + record.setCigarString(ownPrimary.liftedCigar()); + record.setMappingQuality(0); + record.setAttribute(XA_ATTRIBUTE, null); + // SA stays -- rewriteSaTag translates tx-contig entries downstream and FragmentCoords requires it on supps + return true; + } + + public static LiftedMateInfo toLiftedMateInfo(final SAMRecord record, final LiftBackResult result) + { + if(willBeUnmapped(result)) + { + return LiftedMateInfo.UNMAPPED; + } + + Cigar liftedCigar = TextCigarCodec.decode(result.finalCigar()); + int alignmentEnd = result.finalPos() + liftedCigar.getReferenceLength() - 1; + return LiftedMateInfo.mapped(result.finalChrom(), result.finalPos(), alignmentEnd, result.finalCigar(), result.negativeStrand()); + } + + // Mark a previously-mapped primary as unmapped, clearing every per-record tag the SAM spec invariant + // would otherwise leave inconsistent. SA/XA/NH/MC all reference now-stale coords; ProperPair and + // TLEN are meaningless once one end is unmapped. + public static void markPrimaryUnmapped(final SAMRecord record) + { + record.setReadUnmappedFlag(true); + record.setReadNegativeStrandFlag(false); + record.setReferenceName(SAMRecord.NO_ALIGNMENT_REFERENCE_NAME); + record.setAlignmentStart(SAMRecord.NO_ALIGNMENT_START); + record.setCigarString(SAMRecord.NO_ALIGNMENT_CIGAR); + record.setMappingQuality(0); + record.setAttribute(SUPPLEMENTARY_ATTRIBUTE, null); + record.setAttribute(XA_ATTRIBUTE, null); + record.setAttribute("NH", null); + record.setAttribute("MC", null); + if(record.getReadPairedFlag()) + { + record.setProperPairFlag(false); + record.setInferredInsertSize(0); + } + } + + // Single source of truth for whether a primary's final state ends up unmapped, used both at cache- + // build time (so partner records see the correct MateUnmapped) and at apply time (where we mutate + // the record itself). + public static boolean willBeUnmapped(final LiftBackResult result) + { + if(result.recordState() == RecordState.UNMAPPED || result.recordState() == RecordState.LIFT_FAILED) + { + return true; + } + return exceedsMappingCap(result); + } + + // bwa caps the XA alternative-hit list (run with -h 75 here); a read that maps to more loci than the cap + // is emitted with MAPQ 0 and NO XA tag (the list is suppressed, not truncated). Such a read maps to too + // many places to trust, so it is unmapped. inputMapq 0 distinguishes it from a unique read (MAPQ 60, no XA) + // and numXaAlts 0 from an ordinary few-way multimapper (MAPQ 0 with its alts listed in XA). + // + // The cap is meaningful only for a GENOMIC (REF_ONLY) primary: there, 75+ suppressed hits are 75+ distinct + // genomic loci. A tx-contig primary instead hits 75+ transcript contigs of one gene (a shared exon), which + // all lift back to a single genomic locus, so its suppressed XA must NOT be read as "too many genomic + // places". Gating on REF_ONLY keeps genuine genomic repeats unmapped while letting tx reads lift normally. + public static boolean exceedsMappingCap(final LiftBackResult result) + { + return result.inputMapq() == 0 && result.numXaAlts() == 0 && result.comp() == LiftBackResult.Composition.REF_ONLY; + } + + // The tx-contig MD/NM are stale once the read is lifted and (for supplementary-resolve/extend/collapse/canon) recut. + // MD is cigar-coupled and is dropped, not rebuilt: reconstructing it across a spliced read would need + // reference spanning the whole intron. NM only needs the aligned M blocks (N/S/H contribute nothing), + // so it is recomputed cheaply against the genomic reference. With no ref source (supplementary resolve + extend both + // off) or on an unmapped record, NM is cleared as before. + public static void refreshNmDropMd(final SAMRecord record, final RefSequenceSource refSource) + { + record.setAttribute("MD", null); + + if(record.getReadUnmappedFlag() || refSource == null) + { + record.setAttribute("NM", null); + return; + } + + int editDistance = computeEditDistance(record, refSource); + record.setAttribute("NM", editDistance >= 0 ? Integer.valueOf(editDistance) : null); + } + + // NM per the SAM spec: mismatches in aligned blocks plus inserted and deleted bases. N (intron), soft + // and hard clips, and padding contribute nothing, so each M block's reference is fetched on its own and + // the intron is never read. Returns -1 if any block's reference is unavailable, so the caller leaves + // NM unset rather than write a wrong value. + private static int computeEditDistance(final SAMRecord record, final RefSequenceSource refSource) + { + byte[] readBases = record.getReadBases(); + if(readBases == null || readBases.length == 0) + { + return -1; + } + + String chromosome = record.getReferenceName(); + int refPos = record.getAlignmentStart(); + int readIndex = 0; + int editDistance = 0; + + for(final CigarElement element : record.getCigar().getCigarElements()) + { + int length = element.getLength(); + switch(element.getOperator()) + { + case M: + case EQ: + case X: + final byte[] refBases = refSource.getBases(chromosome, refPos, refPos + length - 1); + if(refBases == null || refBases.length < length) + { + return -1; + } + for(int i = 0; i < length; ++i) + { + if(!SequenceUtil.basesEqual(readBases[readIndex + i], refBases[i])) + { + ++editDistance; + } + } + refPos += length; + readIndex += length; + break; + + case I: + editDistance += length; + readIndex += length; + break; + + case D: + editDistance += length; + refPos += length; + break; + + case N: + refPos += length; + break; + + case S: + readIndex += length; + break; + + case H: + case P: + break; + } + } + + return editDistance; + } + + public static String buildLiftedXa(final LiftBackResult result) + { + // Drop alts whose lifted span overlaps the primary choice's span: a tx-contig isoform read in a + // shared exon lifts onto the primary's own coordinates (often a few bp off via softclip-boundary + // or intron-rendering differences, so the exact-coord dedup in LiftBackResolver misses it). Those + // entries re-report the locus bwa already placed the read at and carry no alternative-position + // info. Anything non-overlapping (other locus, other chromosome, a different exon up the same gene) + // is a genuine alternative mapping and is kept. + LiftedAlignment primary = result.liftedAlignments().stream() + .filter(la -> la.IsPrimaryChoice) + .findFirst().orElse(null); + + String xa = result.liftedAlignments().stream() + .filter(la -> !la.IsPrimaryChoice && !la.Dropped) + .filter(la -> !overlapsPrimary(la, primary)) + .map(LiftBackRecordOps::formatXaEntry) + .distinct() // a SELF and a tx-contig alt can lift to identical coords; list each locus once + .collect(Collectors.joining()); + return xa.isEmpty() ? null : xa; + } + + private static boolean overlapsPrimary(final LiftedAlignment alt, final LiftedAlignment primary) + { + if(primary == null || !alt.LiftedChrom.equals(primary.LiftedChrom)) + { + return false; + } + + int altStart = alt.LiftedPos; + int altEnd = altStart + TextCigarCodec.decode(alt.LiftedCigar).getReferenceLength() - 1; + int primStart = primary.LiftedPos; + int primEnd = primStart + TextCigarCodec.decode(primary.LiftedCigar).getReferenceLength() - 1; + return positionsOverlap(altStart, altEnd, primStart, primEnd); + } + + private static String formatXaEntry(final LiftedAlignment la) + { + return la.LiftedChrom + ',' + (la.ForwardStrand ? '+' : '-') + la.LiftedPos + ',' + + la.LiftedCigar + ',' + la.NumMismatches + ';'; + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/LiftBackResolver.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/LiftBackResolver.java new file mode 100644 index 00000000000..8636e3247f3 --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/LiftBackResolver.java @@ -0,0 +1,579 @@ +package com.hartwig.hmftools.tars.liftback; + +import static com.hartwig.hmftools.tars.common.TarsConstants.ALT_CONTIG_SUFFIX; +import static com.hartwig.hmftools.tars.common.TarsConstants.ANNOTATED_JUNCTION_MIN_ANCHOR_BP; +import static com.hartwig.hmftools.tars.common.TarsConstants.ANNOTATED_JUNCTION_MIN_SOFTCLIP_ANCHOR_BP; +import static com.hartwig.hmftools.tars.common.TarsConstants.CONFIDENT_MAPQ; +import static com.hartwig.hmftools.common.bam.SamRecordUtils.XA_ATTRIBUTE; +import static com.hartwig.hmftools.tars.common.TarsConfig.TARS_LOGGER; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import com.hartwig.hmftools.tars.common.ContigEntry; +import com.hartwig.hmftools.tars.common.TarsConstants; + +import htsjdk.samtools.Cigar; +import htsjdk.samtools.SAMRecord; +import htsjdk.samtools.TextCigarCodec; + +// Lifts SAMRecords aligned against ref + Tx contigs to genomic coords. Every input record produces exactly one +// result; UNMAPPED / LIFT_FAILED results carry placeholder fields so the emit step can flag records unmapped. +public class LiftBackResolver +{ + private static final String AS_TAG = "AS"; + private static final String XS_TAG = "XS"; + private static final String NM_TAG = "NM"; + + // per-alt-contig list of segments sorted by altStart for binary search back to the owning transcript. + private final Map> mSegmentsByAltContig; + + // when present, resolves hidden ties (XS==AS, no XA) on ref-only primaries landing inside an annotated exon. + private final ExonRegionIndex mExonIndex; + + public LiftBackResolver(final List entries) + { + this(entries, null); + } + + public LiftBackResolver(final List entries, final ExonRegionIndex exonIndex) + { + mSegmentsByAltContig = new HashMap<>(); + for(final ContigEntry entry : entries) + { + mSegmentsByAltContig.computeIfAbsent(entry.contigName(), k -> new ArrayList<>()).add(entry); + } + for(final List segments : mSegmentsByAltContig.values()) + { + segments.sort(Comparator.comparingInt(ContigEntry::altStart)); + } + + mExonIndex = exonIndex; + } + + public Set contigNames() + { + return mSegmentsByAltContig.keySet(); + } + + // Returns the segment owning altPos, or null if it falls in an inter-transcript spacer or the contig is unknown. + ContigEntry findSegment(final String altContig, final int altPos) + { + List segments = mSegmentsByAltContig.get(altContig); + if(segments == null) + { + return null; + } + + int lo = 0; + int hi = segments.size() - 1; + int candidate = -1; + while(lo <= hi) + { + int mid = (lo + hi) >>> 1; + if(segments.get(mid).altStart() <= altPos) + { + candidate = mid; + lo = mid + 1; + } + else + { + hi = mid - 1; + } + } + + if(candidate < 0) + { + // altPos is before the first segment; return it so ContigTranslator's leading-overhang clamp can salvage it. + return segments.isEmpty() ? null : segments.get(0); + } + + ContigEntry segment = segments.get(candidate); + if(altPos <= segment.altEnd()) + { + return segment; + } + + // altPos in the inter-segment spacer; choose the nearer neighbour and let ContigTranslator clamp the overhang. + if(candidate + 1 < segments.size()) + { + ContigEntry next = segments.get(candidate + 1); + int leadingOverhang = next.altStart() - altPos; + int trailingOverhang = altPos - segment.altEnd(); + return leadingOverhang <= trailingOverhang ? next : segment; + } + + return segment; + } + + // Lift-only entry point for callers that don't need the full LiftBackResult (e.g. SA tag rewriting). + public LiftedCoords liftCoords(final String contig, final int pos, final String cigarStr) + { + LiftedAlignment lifted = liftAlignment( + LiftedAlignment.AlignmentSource.SELF, contig, pos, cigarStr, 0, 0, true); + if(lifted == null) + { + return null; + } + return new LiftedCoords(lifted.LiftedChrom, lifted.LiftedPos, lifted.LiftedCigar); + } + + private LiftedAlignment liftSelf(final SAMRecord record) + { + return liftAlignment( + LiftedAlignment.AlignmentSource.SELF, + record.getReferenceName(), record.getAlignmentStart(), record.getCigarString(), + getInt(record, AS_TAG), getInt(record, NM_TAG), !record.getReadNegativeStrandFlag()); + } + + // Per-call hook to normalize each candidate's lifted cigar (collapse + tail-extend) before the + // discriminator runs, so ref-vs-tx is decided on tested facts. Supplied by the per-worker group + // processor which owns the (thread-unsafe) engines; null on the lift-only / no-ref paths. + @FunctionalInterface + public interface AlignmentNormalizer + { + void normalize(List alignments, SAMRecord record); + } + + // No-reconcile convenience for non-discriminating callers only: supplementaries and unmapped records (which + // skip the discriminator entirely), the lift-only paths, and tests. A primary resolved through here would be + // discriminated on raw bwa cigars, so the production primary path always uses the two-arg form with the + // per-worker reconciler (see LiftBackGroupProcessor.decideMateGroup). + public LiftBackResult resolve(final SAMRecord record) + { + return resolve(record, null); + } + + public LiftBackResult resolve(final SAMRecord record, final AlignmentNormalizer normalizer) + { + if(record.getReadUnmappedFlag()) + { + return unmappedResult(record); + } + + if(record.getSupplementaryAlignmentFlag()) + { + return liftSupplementary(record); + } + + return resolvePrimary(record, normalizer); + } + + private LiftBackResult resolvePrimary(final SAMRecord record, final AlignmentNormalizer normalizer) + { + List xaAlts = parseAndLiftXa(record); + return resolvePrimaryWithAlts(record, xaAlts, xaAlts.size(), normalizer); + } + + private LiftBackResult resolvePrimaryWithAlts( + final SAMRecord record, final List alts, final int numXaAltsForReport, + final AlignmentNormalizer normalizer) + { + LiftedAlignment self = liftSelf(record); + + if(self == null) + { + return unliftableResult(LiftBackResult.RecordRole.PRIMARY, numXaAltsForReport, "primary_translate_failed"); + } + + List allAlignments = new ArrayList<>(1 + alts.size()); + allAlignments.add(self); + allAlignments.addAll(alts); + + // Normalize each candidate's cigar (collapse fabricated micro-junctions, reclaim terminal softclips) + // before discriminating, so RefSoftClipped / RefFullMatch / TxHasNCigar are measured, not assumed. + if(normalizer != null) + { + normalizer.normalize(allAlignments, record); + } + + // normalize() can replace self (index 0) with a reconciled copy, so re-fetch it before marking the + // primary choice. Marking the pre-normalize object would leave IsPrimaryChoice on an orphan no longer + // in the list, and the discriminator's self-identity checks (apply / drop-vs-swap) would misfire. + self = allAlignments.get(0); + self.IsPrimaryChoice = true; + + LiftBackDiscriminator.Features features = LiftBackDiscriminator.categorize(allAlignments); + int inputMapq = record.getMappingQuality(); + int seed = readSeed(record.getReadName()); + LiftBackDiscriminator.ApplyResult outcome = LiftBackDiscriminator.apply( + allAlignments, features.Feature, self, seed, inputMapq != 0); + LiftedAlignment effectivePrimary = outcome.effectivePrimary(); + + List keptAlignments = allAlignments.stream() + .filter(la -> !la.Dropped) + .toList(); + + int numLoci = countDistinctLoci(keptAlignments, effectivePrimary); + int cigarsAtPrimaryLocus = countDistinctCigarsAtLocus(keptAlignments, effectivePrimary); + String geneIds = joinGeneIds(keptAlignments); + + boolean swapped = effectivePrimary != self; + boolean hiddenTie = inputMapq == 0 && hasHiddenTie(record); + boolean inAnnotatedExon = mExonIndex != null + && mExonIndex.contains(effectivePrimary.LiftedChrom, effectivePrimary.LiftedPos); + int updatedMapq = decidePrimaryMapq( + inputMapq, numLoci, hiddenTie, effectivePrimary.fromTxContig(), inAnnotatedExon, features.Feature); + + LiftedAlignment primaryCoords = effectivePrimary; + + if(swapped) + { + TARS_LOGGER.debug2("discriminator {} {}: primary -> {}:{} {} ({})", + features.Feature, record.getReadName(), primaryCoords.LiftedChrom, primaryCoords.LiftedPos, + primaryCoords.LiftedCigar, outcome.note()); + } + + return new LiftBackResult( + RecordState.RESOLVED, features.Feature, swapped, + LiftBackResult.Composition.fromAlignments(keptAlignments), + LiftBackResult.RecordRole.PRIMARY, + primaryCoords.LiftedChrom, primaryCoords.LiftedPos, primaryCoords.LiftedCigar, + !primaryCoords.ForwardStrand, + primaryCoords.cigarHasN(), inputMapq, updatedMapq, + numXaAltsForReport, features.NumRefAlts, features.NumTxAlts, + numLoci, cigarsAtPrimaryLocus, + features.TxHasNCigar, features.TxSoftClipAtBoundary, + features.RefSoftClipped, features.RefFullMatch, + geneIds, outcome.note(), + primaryCoords.TranscriptStrand, + allAlignments); + } + + // A supplementary is only lifted, never discriminated: lift its own coords (no XA parse, no locus pick). + // MAPQ=0 on a tx-contig supplementary is the multi-alt-contig tie artefact; bump to CONFIDENT_MAPQ. + private LiftBackResult liftSupplementary(final SAMRecord record) + { + LiftedAlignment lifted = liftSelf(record); + + if(lifted == null) + { + return unliftableResult(LiftBackResult.RecordRole.SUPPLEMENTARY, 0, "supp_translate_failed"); + } + + lifted.IsPrimaryChoice = true; + + int inputMapq = record.getMappingQuality(); + int outputMapq = (lifted.fromTxContig() && inputMapq == 0) ? CONFIDENT_MAPQ : inputMapq; + int numRefAlts = lifted.fromTxContig() ? 0 : 1; + int numTxAlts = lifted.fromTxContig() ? 1 : 0; + + return new LiftBackResult( + RecordState.SUPPLEMENTARY, null, false, + LiftBackResult.Composition.fromAlignments(List.of(lifted)), + LiftBackResult.RecordRole.SUPPLEMENTARY, + lifted.LiftedChrom, lifted.LiftedPos, lifted.LiftedCigar, + !lifted.ForwardStrand, + lifted.cigarHasN(), inputMapq, outputMapq, + 0, numRefAlts, numTxAlts, + 1, 1, + false, false, false, false, + lifted.GeneId != null ? lifted.GeneId : "", "", + lifted.TranscriptStrand, + List.of(lifted)); + } + + // Self is excluded from the dedup key set so a Tx XA alt lifting to the same coords as a ref self is preserved (drives BOTH_AGREE). + private List parseAndLiftXa(final SAMRecord record) + { + List alts = new ArrayList<>(); + String xa = record.getStringAttribute(XA_ATTRIBUTE); + if(xa == null || xa.isEmpty()) + { + return alts; + } + + Set seenKeys = new HashSet<>(); + + for(final String entry : xa.split(";")) + { + if(entry.isEmpty()) + continue; + String[] parts = entry.split(","); + if(parts.length < 4) + continue; + + String contig = parts[0]; + String cigar = parts[2]; + + // bwa XA pos is signed (sign encodes strand). Tolerate malformed entries. + int signedPos; + int nm; + try + { + signedPos = Integer.parseInt(parts[1]); + } + catch(NumberFormatException e) + { + continue; + } + try + { + nm = Integer.parseInt(parts[3]); + } + catch(NumberFormatException e) + { + nm = 0; + } + + boolean forwardStrand = signedPos >= 0; + int pos = Math.abs(signedPos); + + LiftedAlignment lifted = liftAlignment( + LiftedAlignment.AlignmentSource.XA_INPUT, contig, pos, cigar, 0, nm, forwardStrand); + if(lifted == null) + continue; + if(seenKeys.add(liftedKey(lifted))) + { + alts.add(lifted); + } + } + + return alts; + } + + // Returns null when the contig is an unknown alt contig or the position falls outside any transcript. + // Ref alignments pass through as-is. + private LiftedAlignment liftAlignment( + final LiftedAlignment.AlignmentSource source, final String contig, final int pos, final String cigarStr, + final int as, final int nm, final boolean forwardStrand) + { + if(!mSegmentsByAltContig.containsKey(contig)) + { + // Unknown alt contig must not pass through as ref - that would leak _tx contig names into the BAM. + if(contig.endsWith(ALT_CONTIG_SUFFIX)) + { + return null; + } + + return new LiftedAlignment( + source, contig, pos, cigarStr, + contig, pos, cigarStr, + as, nm, + null, null, null, + false, forwardStrand); + } + + ContigEntry entry = findSegment(contig, pos); + if(entry == null) + { + return null; + } + + Cigar parsedCigar = TextCigarCodec.decode(cigarStr); + ContigTranslator.TranslationResult translated = ContigTranslator.translate(entry, pos, parsedCigar); + if(translated == null) + { + return null; + } + + boolean softClipAtBoundary = ContigTranslator.hasSoftClipAtExonBoundary(entry, pos, parsedCigar); + + // Drop sub-threshold terminal anchors that bwa fabricated by walking past an exon boundary. + ContigTranslator.MicroAnchorResult trimmed = ContigTranslator.trimMicroAnchors( + translated.genomicCigar(), ANNOTATED_JUNCTION_MIN_ANCHOR_BP, ANNOTATED_JUNCTION_MIN_SOFTCLIP_ANCHOR_BP); + + return new LiftedAlignment( + source, contig, pos, cigarStr, + translated.chromosome(), translated.genomicStart() + trimmed.StartShift, + trimmed.AdjustedCigar.toString(), + as, nm, + entry.transName(), entry.geneId(), entry.geneName(), + softClipAtBoundary, forwardStrand, entry.strand()); + } + + // Counts distinct genomic loci among the kept alignments, using the SAME same-locus test as buildLiftedXa + // (overlapsPrimary) so NH and the emitted XA always agree. The primary is one locus; an alt whose genomic span + // OVERLAPS the primary's is the same locus and collapses -- a 5'/3'-softclipped isoform copy begins at a + // downstream exon of the same placement, lifting to a different start but a span nested inside the primary's, so + // keying on exact start would over-count it and wrongly withhold the MAPQ bump. Alts that do NOT overlap the + // primary are genuinely distinct placements (repeats/paralogs, other chromosomes); they are interval-merged + // among themselves so a cluster of overlapping repeat copies at one other locus counts once, but a tandem-repeat + // array offset beyond the primary is NOT chained back into the primary (that would falsely bump a repeat + // multimapper). numLoci therefore equals 1 exactly when every alt overlaps the primary -- i.e. when buildLiftedXa + // emits no XA -- so a bumped MAPQ-60 / NH-1 read never carries a stale XA. + private static int countDistinctLoci(final List alignments, final LiftedAlignment primary) + { + int primaryEnd = spanEnd(primary); + + Map> distinctSpans = new HashMap<>(); + distinctSpans.computeIfAbsent(primary.LiftedChrom, k -> new ArrayList<>()) + .add(new int[] { primary.LiftedPos, primaryEnd }); + + for(final LiftedAlignment la : alignments) + { + if(la == primary) + { + continue; + } + int end = spanEnd(la); + boolean overlapsPrimary = la.LiftedChrom.equals(primary.LiftedChrom) + && la.LiftedPos <= primaryEnd && primary.LiftedPos <= end; + if(overlapsPrimary) + { + continue; + } + distinctSpans.computeIfAbsent(la.LiftedChrom, k -> new ArrayList<>()).add(new int[] { la.LiftedPos, end }); + } + + int loci = 0; + for(final List spans : distinctSpans.values()) + { + spans.sort(Comparator.comparingInt(s -> s[0])); + int clusterEnd = -1; + for(final int[] span : spans) + { + if(span[0] > clusterEnd) + { + ++loci; + clusterEnd = span[1]; + } + else + { + clusterEnd = Math.max(clusterEnd, span[1]); + } + } + } + return loci; + } + + private static int spanEnd(final LiftedAlignment la) + { + return la.LiftedPos + TextCigarCodec.decode(la.LiftedCigar).getReferenceLength() - 1; + } + + private static int countDistinctCigarsAtLocus(final List alignments, final LiftedAlignment primary) + { + String primaryLocus = locusKey(primary); + Set cigars = new HashSet<>(); + for(final LiftedAlignment la : alignments) + { + if(locusKey(la).equals(primaryLocus)) + { + cigars.add(la.LiftedCigar); + } + } + return cigars.size(); + } + + private static String joinGeneIds(final List alignments) + { + Set geneIds = new HashSet<>(); + for(final LiftedAlignment la : alignments) + if(la.GeneId != null) + { + geneIds.add(la.GeneId); + } + return geneIds.stream().sorted().collect(Collectors.joining("|")); + } + + private static String liftedKey(final LiftedAlignment la) + { + return la.LiftedChrom + ":" + la.LiftedPos + ":" + la.LiftedCigar; + } + + private static String locusKey(final LiftedAlignment la) + { + return la.LiftedChrom + ":" + la.LiftedPos; + } + + private static int getInt(final SAMRecord record, final String tag) + { + Integer val = record.getIntegerAttribute(tag); + return val != null ? val : 0; + } + + private static LiftBackResult unmappedResult(final SAMRecord record) + { + LiftBackResult.RecordRole role = record.isSecondaryOrSupplementary() + ? LiftBackResult.RecordRole.SUPPLEMENTARY + : LiftBackResult.RecordRole.PRIMARY; + + return new LiftBackResult( + RecordState.UNMAPPED, null, false, LiftBackResult.Composition.NONE, + role, + "*", 0, "*", + false, + false, 0, 0, + 0, 0, 0, + 0, 0, + false, false, false, false, + "", "", + 0, + List.of()); + } + + // Bump a MAPQ-0 primary to 60 only when it lifts to a single locus with no competing alt. Multi-locus reads + // (including a discriminator swap to an alt at another locus) are never promoted. A hidden tie (XS==AS, an + // equal-scoring alt bwa did not emit) is a latent second locus and blocks the bump unless tx provenance or an + // annotated exon vouches for the placement. + static int decidePrimaryMapq( + final int inputMapq, final int numLoci, final boolean hiddenTie, + final boolean primaryFromTxContig, final boolean primaryInAnnotatedExon) + { + return decidePrimaryMapq(inputMapq, numLoci, hiddenTie, primaryFromTxContig, primaryInAnnotatedExon, null); + } + + static int decidePrimaryMapq( + final int inputMapq, final int numLoci, final boolean hiddenTie, + final boolean primaryFromTxContig, final boolean primaryInAnnotatedExon, final DecidingFeature feature) + { + if(inputMapq != 0) + { + return inputMapq; + } + // an ambiguous single-locus call is a coin-flip between tx and ref, so it must never be promoted to confident. + if(feature == DecidingFeature.AMBIGUOUS) + { + return inputMapq; + } + if(numLoci != 1) + { + return inputMapq; + } + boolean unresolvedHiddenTie = hiddenTie && !primaryFromTxContig && !primaryInAnnotatedExon; + if(unresolvedHiddenTie) + { + return inputMapq; + } + return CONFIDENT_MAPQ; + } + + // Per-read deterministic seed for the not-confident random placement picks. Both mates share a read name, + // so a pair is placed together, and the value is stable across runs, keeping the output reproducible. + static int readSeed(final String readName) + { + int hash = readName.hashCode(); + return hash ^ (hash >>> 16); + } + + // When XS == AS, an equally-scoring alt wasn't emitted by bwa. Flag as a hidden tie to skip the MAPQ bump. + private static boolean hasHiddenTie(final SAMRecord record) + { + Integer alignmentScore = record.getIntegerAttribute(AS_TAG); + Integer suboptimalScore = record.getIntegerAttribute(XS_TAG); + return alignmentScore != null && suboptimalScore != null && suboptimalScore.intValue() == alignmentScore.intValue(); + } + + private static LiftBackResult unliftableResult(final LiftBackResult.RecordRole role, final int numXaAlts, final String note) + { + return new LiftBackResult( + RecordState.LIFT_FAILED, null, false, LiftBackResult.Composition.NONE, + role, + "*", 0, "*", + false, + false, 0, 0, + numXaAlts, 0, 0, + 0, 0, + false, false, false, false, + "", note, + 0, + List.of()); + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/LiftBackResources.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/LiftBackResources.java new file mode 100644 index 00000000000..0b125c9ac85 --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/LiftBackResources.java @@ -0,0 +1,73 @@ +package com.hartwig.hmftools.tars.liftback; + +import static com.hartwig.hmftools.tars.common.TarsConfig.TARS_LOGGER; + +import java.io.File; + +import com.hartwig.hmftools.tars.liftback.supplementary.AnnotatedJunctionIndex; +import com.hartwig.hmftools.tars.liftback.supplementary.RefSequenceSource; +import com.hartwig.hmftools.tars.liftback.supplementary.SupplementaryConfig; + +import htsjdk.samtools.reference.IndexedFastaSequenceFile; +import htsjdk.samtools.reference.ReferenceSequence; + +// Shared read-only inputs handed to every LiftBackWorker: the sidecar resolver and junction index hold no +// mutable state, so all workers share one instance. Each worker builds its own engines + ref-source handle +// from this (RefSequenceSource and the supplementary-resolve/tail-extend stats are not thread-safe). +public final class LiftBackResources +{ + public final LiftBackResolver Resolver; + public final AnnotatedJunctionIndex JunctionIndex; // nullable + public final String RefGenomeFile; + public final SupplementaryConfig Supplementary; + public final ExcludedRegions ExcludedRegions; // nullable: drop fragments here before lifting + + public LiftBackResources( + final LiftBackResolver resolver, final AnnotatedJunctionIndex junctionIndex, final String refGenomeFile, + final SupplementaryConfig supplementary, final ExcludedRegions excludedRegions) + { + Resolver = resolver; + JunctionIndex = junctionIndex; + RefGenomeFile = refGenomeFile; + Supplementary = supplementary; + ExcludedRegions = excludedRegions; + } + + // one handle per caller. IndexedFastaSequenceFile is not thread-safe, so workers each open their own. + public RefSequenceSource openRefSource() + { + return openRefSource(RefGenomeFile); + } + + public static RefSequenceSource openRefSource(final String refGenomeFile) + { + if(refGenomeFile == null) + { + return null; + } + try + { + IndexedFastaSequenceFile fasta = new IndexedFastaSequenceFile(new File(refGenomeFile)); + return (chromosome, posStart, posEnd) -> + { + synchronized (fasta) + { + try + { + ReferenceSequence seq = fasta.getSubsequenceAt(chromosome, posStart, posEnd); + return seq != null ? seq.getBases() : null; + } + catch(Exception e) + { + return null; + } + } + }; + } + catch(Exception e) + { + TARS_LOGGER.warn("could not open ref FASTA {} for ref-verify: {}", refGenomeFile, e.toString()); + return null; + } + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/LiftBackResult.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/LiftBackResult.java new file mode 100644 index 00000000000..1630fc1d750 --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/LiftBackResult.java @@ -0,0 +1,136 @@ +package com.hartwig.hmftools.tars.liftback; + +import java.util.List; + +// Per-record decision result. Drives the BAM record rewrite and one TSV-A row. +// recordState says whether the record reached the discriminator; for RESOLVED records outcome/decidingFeature +// carry the ref-vs-tx decision and its reason. liftedAlignments holds the full alignment set (self + lifted +// XA alts) for TSV-B. +public record LiftBackResult( + RecordState recordState, + DecidingFeature decidingFeature, + boolean swapped, + Composition comp, + RecordRole role, + String finalChrom, + int finalPos, + String finalCigar, + boolean negativeStrand, + boolean hasNCigar, + int inputMapq, + int updatedMapq, + int numXaAlts, + int numRefAlts, + int numTxAlts, + int numLoci, + int numDistinctCigarsAtPrimaryLocus, + boolean txHasNCigar, + boolean txSoftClipAtBoundary, + boolean refSoftClipped, + boolean refFullMatch, + String geneIds, + String notes, + // +1/−1 for tx-contig-derived primaries; 0 for ref-only or supplementary-resolve/tail-extend origins. + // Used by the writer to set XS:A:+/- on spliced records. + int transcriptStrand, + List liftedAlignments) +{ + // Outcome is derived from the deciding feature rather than stored alongside it; non-decision records + // (null feature: unmapped / lift-failed / supplementary) are UNRESOLVED. + public Outcome outcome() + { + return decidingFeature != null ? decidingFeature.outcome() : Outcome.UNRESOLVED; + } + + // Copy with a post-lift cigar pass applied: position, cigar, N-flag, MAPQ and an appended note change; + // every other field is preserved. Shared by the supplementary-resolve / collapse / tail-extend / canonicalize passes. + public LiftBackResult withRevisedCigar( + final int newPos, final String newCigar, final boolean newHasNCigar, final int newUpdatedMapq, final String note) + { + return new LiftBackResult( + recordState, decidingFeature, swapped, comp, role, + finalChrom, newPos, newCigar, + negativeStrand, newHasNCigar, + inputMapq, newUpdatedMapq, + numXaAlts, numRefAlts, numTxAlts, + numLoci, numDistinctCigarsAtPrimaryLocus, + txHasNCigar, txSoftClipAtBoundary, + refSoftClipped, refFullMatch, + geneIds, + appendNote(notes, note), + transcriptStrand, + liftedAlignments); + } + + // UNMAPPED result with the given role and a diagnostic note. Used when a lifted placement is rejected + // post-lift (e.g. it falls in an excluded region) and the record must be flipped to unmapped. + public static LiftBackResult unmapped(final RecordRole role, final String note) + { + return new LiftBackResult( + RecordState.UNMAPPED, null, false, Composition.NONE, role, + "*", 0, "*", + false, false, 0, 0, + 0, 0, 0, + 0, 0, + false, false, false, false, + "", note, + 0, + List.of()); + } + + private static String appendNote(final String existing, final String note) + { + if(existing == null || existing.isEmpty()) + { + return note; + } + return existing + ";" + note; + } + + public enum RecordRole + { + PRIMARY, + SUPPLEMENTARY + } + + // post-drop view drives TSV-A + XA rebuild; pre-drop view drives LiftBackStats summary + public enum Composition + { + REF_ONLY, + TX_ONLY, + REF_AND_TX, + NONE; + + public static Composition fromAlignments(final List alignments) + { + if(alignments.isEmpty()) + { + return NONE; + } + + boolean hasRef = false; + boolean hasTx = false; + for(final LiftedAlignment alignment : alignments) + { + if(alignment.fromTxContig()) + { + hasTx = true; + } + else + { + hasRef = true; + } + } + + if(hasRef && hasTx) + { + return REF_AND_TX; + } + if(hasTx) + { + return TX_ONLY; + } + return REF_ONLY; + } + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/LiftBackStats.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/LiftBackStats.java new file mode 100644 index 00000000000..fe8d64f33c9 --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/LiftBackStats.java @@ -0,0 +1,344 @@ +package com.hartwig.hmftools.tars.liftback; + +import static com.hartwig.hmftools.common.utils.file.FileDelimiters.TSV_DELIM; +import static com.hartwig.hmftools.common.utils.file.FileWriterUtils.createBufferedWriter; +import static com.hartwig.hmftools.tars.common.TarsConstants.PRIMARY_AS_UNMAP_THRESHOLD; +import static com.hartwig.hmftools.tars.common.TarsConstants.SUPP_AS_DROP_THRESHOLD; +import static com.hartwig.hmftools.tars.common.TarsConfig.TARS_LOGGER; + +import java.io.BufferedWriter; +import java.io.IOException; + +import htsjdk.samtools.SAMRecord; + +// Run-level counters for TarsApplication: per deciding-feature, per record-state, composition x MAPQ tier, +// deciding-feature x MAPQ tier. Non-RESOLVED records (unmapped / lift-failed / supplementary) carry no +// deciding feature, so they are counted only in the record-state tally. +public class LiftBackStats +{ + private static final double LIFT_FAILED_WARN_FRACTION = 0.01; + + public enum MapqTier + { + MAPQ_ZERO, // bwa flagged as multi-mapping + MAPQ_POS_MULTI, // MAPQ > 0 but XA non-empty (some ambiguity remains) + MAPQ_POS_UNIQUE // MAPQ > 0, no XA alts + } + + private static final int N_FEATURES = DecidingFeature.values().length; + private static final int N_STATES = RecordState.values().length; + private static final int N_COMPOSITIONS = LiftBackResult.Composition.values().length; + private static final int N_TIERS = MapqTier.values().length; + + private final int[] mPerFeature = new int[N_FEATURES]; + private final int[] mSwapByFeature = new int[N_FEATURES]; + private final int[] mPerState = new int[N_STATES]; + private final int[][] mCompositionByTier = new int[N_COMPOSITIONS][N_TIERS]; + private final int[][] mFeatureByTier = new int[N_FEATURES][N_TIERS]; + private int mTotal = 0; + private int mSplicedOutput = 0; // resolved records emitted with an N (spliced) + private int mMapqZeroIn = 0; // bwa MAPQ 0 on a resolved primary + private int mMapqRescued = 0; // of those, lifted to a positive MAPQ + private int mLowAsSuppsDropped = 0; + private int mOrphanSuppsDropped = 0; + private int mLowAsPrimariesUnmapped = 0; + private PassEffects mPassEffects = PassEffects.EMPTY; + + // Per-pass effectiveness, aggregated across workers by the caller and folded into the summary so the + // always-written summary answers "what did liftback do" without the opt-in ~100GB per-record TSV. + public record PassEffects( + int suppCandidates, int suppMerged, int suppClamped, + long collapseLeading, long collapseTrailing, + long reclaimRecords, long reclaimBasesLead, long reclaimBasesTrail, long altsDropped, + long overCapUnmapped, long excludedReads) + { + public static final PassEffects EMPTY = new PassEffects(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + } + + public void setPassEffects(final PassEffects passEffects) + { + mPassEffects = passEffects; + } + + public void record(final SAMRecord record, final LiftBackResult result) + { + // Full pre-drop composition (includes discriminator-dropped alts) reflects what bwa produced. + LiftBackResult.Composition fullComposition = LiftBackResult.Composition.fromAlignments(result.liftedAlignments()); + MapqTier tier = deriveMapqTier(record, result); + + ++mTotal; + ++mCompositionByTier[fullComposition.ordinal()][tier.ordinal()]; + ++mPerState[result.recordState().ordinal()]; + + DecidingFeature feature = result.decidingFeature(); + if(feature != null) // RESOLVED records only + { + ++mPerFeature[feature.ordinal()]; + ++mFeatureByTier[feature.ordinal()][tier.ordinal()]; + if(result.swapped()) + { + ++mSwapByFeature[feature.ordinal()]; + } + if(result.hasNCigar()) + { + ++mSplicedOutput; + } + if(result.inputMapq() == 0) + { + ++mMapqZeroIn; + if(result.updatedMapq() > 0) + { + ++mMapqRescued; + } + } + } + } + + public int total() + { + return mTotal; + } + + public int featureCount(final DecidingFeature feature) + { + return mPerFeature[feature.ordinal()]; + } + + public int stateCount(final RecordState state) + { + return mPerState[state.ordinal()]; + } + + public void recordLowAsSuppDropped() + { + ++mLowAsSuppsDropped; + } + + public int lowAsSuppsDropped() + { + return mLowAsSuppsDropped; + } + + public void recordOrphanSuppDropped() + { + ++mOrphanSuppsDropped; + } + + public int orphanSuppsDropped() + { + return mOrphanSuppsDropped; + } + + public void recordLowAsPrimaryUnmapped() + { + ++mLowAsPrimariesUnmapped; + } + + public int lowAsPrimariesUnmapped() + { + return mLowAsPrimariesUnmapped; + } + + // Merge per-worker stats into this instance for a combined end-of-run summary. + public void merge(final LiftBackStats other) + { + mTotal += other.mTotal; + mSplicedOutput += other.mSplicedOutput; + mMapqZeroIn += other.mMapqZeroIn; + mMapqRescued += other.mMapqRescued; + mLowAsSuppsDropped += other.mLowAsSuppsDropped; + mOrphanSuppsDropped += other.mOrphanSuppsDropped; + mLowAsPrimariesUnmapped += other.mLowAsPrimariesUnmapped; + for(int i = 0; i < N_FEATURES; ++i) + { + mPerFeature[i] += other.mPerFeature[i]; + mSwapByFeature[i] += other.mSwapByFeature[i]; + } + for(int i = 0; i < N_STATES; ++i) + { + mPerState[i] += other.mPerState[i]; + } + for(int i = 0; i < mCompositionByTier.length; ++i) + for(int j = 0; j < N_TIERS; ++j) + { + mCompositionByTier[i][j] += other.mCompositionByTier[i][j]; + } + for(int i = 0; i < N_FEATURES; ++i) + for(int j = 0; j < N_TIERS; ++j) + { + mFeatureByTier[i][j] += other.mFeatureByTier[i][j]; + } + } + + // Outcome counts are derived from the per-feature tallies (outcome is a function of the deciding feature), + // so they cover resolved primaries only - matching the old per-outcome counter. + private int outcomeCount(final Outcome outcome) + { + int total = 0; + for(DecidingFeature feature : DecidingFeature.values()) + { + if(feature.outcome() == outcome) + { + total += mPerFeature[feature.ordinal()]; + } + } + return total; + } + + public void logSummary() + { + TARS_LOGGER.info("processed {} records", mTotal); + TARS_LOGGER.info("resolved outcomes: REF={} TX={} UNRESOLVED={}", + outcomeCount(Outcome.REF), outcomeCount(Outcome.TX), outcomeCount(Outcome.UNRESOLVED)); + TARS_LOGGER.info("spliced output reads: {}; MAPQ-0 in: {}, rescued: {}", mSplicedOutput, mMapqZeroIn, mMapqRescued); + if(mLowAsSuppsDropped > 0) + { + TARS_LOGGER.info("dropped {} non-resolved supps with source AS < {}", + mLowAsSuppsDropped, SUPP_AS_DROP_THRESHOLD); + } + if(mOrphanSuppsDropped > 0) + { + TARS_LOGGER.info("dropped {} supps whose SA entries all failed to lift (orphaned in genomic space)", + mOrphanSuppsDropped); + } + if(mLowAsPrimariesUnmapped > 0) + { + TARS_LOGGER.info("unmapped {} primaries with AS < {} not improved by liftback", + mLowAsPrimariesUnmapped, PRIMARY_AS_UNMAP_THRESHOLD); + } + + TARS_LOGGER.debug("alignment-set composition x MAPQ tier:"); + logTable(mCompositionByTier, LiftBackResult.Composition.values(), MapqTier.values()); + + TARS_LOGGER.debug("deciding-feature x MAPQ tier:"); + logTable(mFeatureByTier, DecidingFeature.values(), MapqTier.values()); + + TARS_LOGGER.debug("record-state breakdown:"); + for(RecordState state : RecordState.values()) + { + int count = mPerState[state.ordinal()]; + if(count > 0) + { + TARS_LOGGER.debug(" {}={}", state.name(), count); + } + } + + int unliftable = mPerState[RecordState.LIFT_FAILED.ordinal()]; + if(mTotal > 0 && unliftable / (double) mTotal > LIFT_FAILED_WARN_FRACTION) + { + // ERROR not throw: BAM/TSVs already written should remain available for inspection. + TARS_LOGGER.error("LIFT_FAILED rate {} / {} = {}% exceeds {}% threshold - likely sidecar/FASTA mismatch", + unliftable, mTotal, + String.format("%.2f", 100.0 * unliftable / mTotal), + String.format("%.2f", 100.0 * LIFT_FAILED_WARN_FRACTION)); + } + } + + // Flat, greppable one-metric-per-line summary: Metric, Value, Pct, Basis. Pct is Value as a percentage + // of the named Basis metric so each rate is self-documenting (grep the metric, read the number and what + // it is a fraction of). The 2D cross-tabs (composition/feature x MAPQ tier) stay in the DEBUG log only. + public void writeSummary(final String path) throws IOException + { + long resolved = mPerState[RecordState.RESOLVED.ordinal()]; + long liftFailed = mPerState[RecordState.LIFT_FAILED.ordinal()]; + long swaps = 0; + for(int i = 0; i < N_FEATURES; ++i) + { + swaps += mSwapByFeature[i]; + } + + try(BufferedWriter writer = createBufferedWriter(path)) + { + writer.write(String.join(TSV_DELIM, "Metric", "Value", "Pct", "Basis")); + writer.newLine(); + + // records read, the primaries the discriminator placed (the denominator for the splits below), and + // the lift-failure count - a non-zero rate here flags a sidecar/FASTA mismatch. + writeMetric(writer, "records_total", mTotal); + writeMetric(writer, "primaries_resolved", resolved, "records_total", mTotal); + writeMetric(writer, "lift_failed", liftFailed, "records_total", mTotal); + + // discriminator outcome and the feature that decided it, over resolved primaries. + for(Outcome outcome : Outcome.values()) + { + writeMetric(writer, "outcome_" + outcome.name().toLowerCase(), outcomeCount(outcome), "primaries_resolved", resolved); + } + for(DecidingFeature feature : DecidingFeature.values()) + { + writeMetric(writer, "feature_" + feature.name().toLowerCase(), mPerFeature[feature.ordinal()], "primaries_resolved", resolved); + } + writeMetric(writer, "primary_swapped", swaps, "primaries_resolved", resolved); + + // MAPQ rescue and splicing. + writeMetric(writer, "mapq_zero_in", mMapqZeroIn, "primaries_resolved", resolved); + writeMetric(writer, "mapq_rescued", mMapqRescued, "mapq_zero_in", mMapqZeroIn); + writeMetric(writer, "spliced_output", mSplicedOutput, "primaries_resolved", resolved); + + // refinement passes: what each edit actually changed (leading/trailing merged into one figure). + writeMetric(writer, "supp_merged", mPassEffects.suppMerged()); + writeMetric(writer, "supp_clamped", mPassEffects.suppClamped()); + writeMetric(writer, "overhang_reclaim_records", mPassEffects.reclaimRecords()); + writeMetric(writer, "overhang_reclaim_bases", mPassEffects.reclaimBasesLead() + mPassEffects.reclaimBasesTrail()); + writeMetric(writer, "overhang_collapse", mPassEffects.collapseLeading() + mPassEffects.collapseTrailing()); + writeMetric(writer, "overhang_alt_dropped", mPassEffects.altsDropped()); + + // records removed from output. + writeMetric(writer, "excluded_region_reads", mPassEffects.excludedReads()); + writeMetric(writer, "supp_dropped_low_as", mLowAsSuppsDropped); + writeMetric(writer, "supp_dropped_orphan", mOrphanSuppsDropped); + writeMetric(writer, "primary_unmapped_low_as", mLowAsPrimariesUnmapped); + writeMetric(writer, "primary_unmapped_over_cap", mPassEffects.overCapUnmapped()); + } + + TARS_LOGGER.info("wrote summary to {}", path); + } + + private static void writeMetric(final BufferedWriter writer, final String metric, final long value) throws IOException + { + writer.write(String.join(TSV_DELIM, metric, String.valueOf(value), "", "")); + writer.newLine(); + } + + private static void writeMetric( + final BufferedWriter writer, final String metric, final long value, final String basis, final long basisValue) + throws IOException + { + String pct = basisValue > 0 ? String.format("%.2f", 100.0 * value / basisValue) : ""; + writer.write(String.join(TSV_DELIM, metric, String.valueOf(value), pct, basis)); + writer.newLine(); + } + + private static , C extends Enum> void logTable( + final int[][] table, final R[] rows, final C[] cols) + { + for(R row : rows) + { + StringBuilder sb = new StringBuilder(); + int rowTotal = 0; + for(C col : cols) + { + int count = table[row.ordinal()][col.ordinal()]; + rowTotal += count; + if(count > 0) + { + sb.append(col.name()).append("=").append(count).append(" "); + } + } + + if(rowTotal == 0) + continue; + + TARS_LOGGER.debug(" {}: total={} {}", row.name(), rowTotal, sb.toString()); + } + } + + static MapqTier deriveMapqTier(final SAMRecord record, final LiftBackResult result) + { + int mapq = record.getMappingQuality(); + if(mapq == 0) + { + return MapqTier.MAPQ_ZERO; + } + return result.numXaAlts() > 0 ? MapqTier.MAPQ_POS_MULTI : MapqTier.MAPQ_POS_UNIQUE; + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/LiftBackWorker.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/LiftBackWorker.java new file mode 100644 index 00000000000..53d3988113f --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/LiftBackWorker.java @@ -0,0 +1,169 @@ +package com.hartwig.hmftools.tars.liftback; + +import static com.hartwig.hmftools.tars.common.TarsConfig.TARS_LOGGER; +import static com.hartwig.hmftools.tars.liftback.ChunkProducer.END_OF_STREAM; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.BlockingQueue; + +import com.hartwig.hmftools.tars.liftback.overhang.OverhangGate; +import com.hartwig.hmftools.tars.liftback.overhang.OverhangGateStatistics; +import com.hartwig.hmftools.tars.liftback.supplementary.SupplementaryResolver; +import com.hartwig.hmftools.tars.liftback.supplementary.RefSequenceSource; +import com.hartwig.hmftools.tars.liftback.supplementary.SupplementaryStatistics; + +import htsjdk.samtools.SAMFileHeader; +import htsjdk.samtools.SAMFileWriter; +import htsjdk.samtools.SAMFileWriterFactory; +import htsjdk.samtools.SAMRecord; + +// Drains read-name chunks from the shared queue, transforms each contiguous name-group with a per-group +// LiftedMateInfoCache, and writes lifted records to its own unsorted shard. Owns its engines and ref-source +// handle so nothing serialises across workers. Exits on END_OF_STREAM. +public class LiftBackWorker extends Thread +{ + private final BlockingQueue> mQueue; + private final LiftBackStats mStats; + private final LiftBackGroupProcessor mProcessor; + private final SAMFileWriter mShardWriter; + private final LiftBackWriter mTsvWriter; // nullable: headerless per-worker TSV shard, only when enabled + + private final SupplementaryResolver mSupplementaryResolver; + private final OverhangGate mOverhangGate; + + private final ExcludedRegions mExcludedRegions; // nullable: passed to the processor for post-lift exclusion + + public LiftBackWorker( + final BlockingQueue> queue, final LiftBackResources resources, + final SAMFileHeader header, final String shardBam, final String tsvAShard, final String tsvBShard) + { + mQueue = queue; + mStats = new LiftBackStats(); + mExcludedRegions = resources.ExcludedRegions; + + try + { + mTsvWriter = tsvAShard != null ? new LiftBackWriter(tsvAShard, tsvBShard, false) : null; + } + catch(IOException e) + { + throw new RuntimeException("failed to open liftback TSV shard: " + e, e); + } + + RefSequenceSource refSource = resources.openRefSource(); + + mSupplementaryResolver = new SupplementaryResolver(resources.JunctionIndex, refSource, resources.Supplementary); + mOverhangGate = new OverhangGate(refSource); + + mProcessor = new LiftBackGroupProcessor( + resources.Resolver, mSupplementaryResolver, mOverhangGate, + refSource, mExcludedRegions, mStats); + + mShardWriter = new SAMFileWriterFactory().makeBAMWriter(header, false, new File(shardBam)); + } + + public LiftBackStats liftBackStats() { return mStats; } + + public SupplementaryStatistics supplementaryStatistics() + { + return mSupplementaryResolver != null ? mSupplementaryResolver.statistics() : null; + } + + public OverhangGateStatistics overhangStatistics() { return mOverhangGate != null ? mOverhangGate.statistics() : null; } + + public long overCapUnmapped() { return mProcessor.overCapUnmapped(); } + + @Override + public void run() + { + try + { + while(true) + { + List chunk = mQueue.take(); + if(chunk == END_OF_STREAM) + break; + processChunk(chunk); + } + } + catch(InterruptedException e) + { + Thread.currentThread().interrupt(); + TARS_LOGGER.error("liftback worker interrupted: {}", e.toString()); + System.exit(1); + } + catch(Exception e) + { + TARS_LOGGER.error("liftback worker failed: {}", e.toString(), e); + System.exit(1); + } + finally + { + mShardWriter.close(); + if(mTsvWriter != null) + { + try + { + mTsvWriter.close(); + } + catch(IOException e) + { + TARS_LOGGER.warn("failed to close liftback TSV shard: {}", e.toString()); + } + } + } + } + + private void processChunk(final List chunk) + { + List group = new ArrayList<>(); + String currentName = null; + + for(final SAMRecord record : chunk) + { + String name = record.getReadName(); + if(currentName != null && !name.equals(currentName)) + { + processNameGroup(group); + group.clear(); + } + group.add(record); + currentName = name; + } + + if(!group.isEmpty()) + { + processNameGroup(group); + } + } + + private void processNameGroup(final List group) + { + // exclusion is applied post-lift inside the processor (on lifted genomic coords): a primary lifting into + // an excluded region is unmapped REDUX-style, a supp dropped. Pre-lift can't test tx-contig reads, whose + // input coords are chrN_tx, against the genomic region list. + mProcessor.processNameGroup(group, new LiftedMateInfoCache(), this::write); + } + + public long excludedReads() { return mProcessor.excludedReads(); } + + private void write(final SAMRecord record, final LiftBackResult result) + { + mShardWriter.addAlignment(record); + if(mTsvWriter == null) + { + return; + } + try + { + mTsvWriter.write(record, result); + } + catch(IOException e) + { + throw new RuntimeException("failed to write liftback TSV shard: " + e, e); + } + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/LiftBackWriter.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/LiftBackWriter.java new file mode 100644 index 00000000000..00e6a8435d4 --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/LiftBackWriter.java @@ -0,0 +1,146 @@ +package com.hartwig.hmftools.tars.liftback; + +import static com.hartwig.hmftools.common.utils.file.FileDelimiters.TSV_DELIM; +import static com.hartwig.hmftools.common.utils.file.FileWriterUtils.createBufferedWriter; + +import java.io.BufferedWriter; +import java.io.IOException; + +import htsjdk.samtools.SAMRecord; + +public class LiftBackWriter implements AutoCloseable +{ + private final BufferedWriter mTsvAWriter; + private final BufferedWriter mTsvBWriter; + + private static final String[] TSV_A_HEADERS = { + "ReadName", "MateNum", "RecordRole", "RecordState", "Composition", "Outcome", "DecidingFeature", "Swapped", + "InputMapq", "UpdatedMapq", + "NumXaAlts", "NumRefAlts", "NumTxAlts", "NumLoci", "NumDistinctCigarsAtPrimaryLocus", + "TxHasNCigar", "TxSoftClipAtBoundary", "RefSoftClipped", "RefFullMatch", + "FinalChrom", "FinalPos", "FinalCigar", "HasNCigar", + "GeneIds", "Notes" + }; + + private static final String[] TSV_B_HEADERS = { + "ReadName", "MateNum", "RecordRole", "Source", + "OrigContig", "OrigPos", "OrigCigar", + "LiftedChrom", "LiftedPos", "LiftedCigar", + "AlignmentScore", "NumMismatches", + "TransName", "GeneId", "GeneName" + }; + + // header lines for the records (A) and alignments (B) TSVs, written once by the shard-concat step. + public static final String TSV_A_HEADER_LINE = String.join(TSV_DELIM, TSV_A_HEADERS); + public static final String TSV_B_HEADER_LINE = String.join(TSV_DELIM, TSV_B_HEADERS); + + public LiftBackWriter(final String tsvAPath, final String tsvBPath) throws IOException + { + this(tsvAPath, tsvBPath, true); + } + + // writeHeaders=false produces a data-only shard: each worker writes one, and the driver concatenates the + // shards under a single header line (see TarsApplication.concatenateTsvShards). + public LiftBackWriter(final String tsvAPath, final String tsvBPath, final boolean writeHeaders) throws IOException + { + mTsvAWriter = createBufferedWriter(tsvAPath); + mTsvBWriter = createBufferedWriter(tsvBPath); + if(writeHeaders) + { + writeHeader(mTsvAWriter, TSV_A_HEADERS); + writeHeader(mTsvBWriter, TSV_B_HEADERS); + } + } + + public void write(final SAMRecord record, final LiftBackResult result) throws IOException + { + String readName = record.getReadName(); + String mateNum = mateNumColumn(record); + writeTsvARow(readName, mateNum, result); + for(final LiftedAlignment la : result.liftedAlignments()) + { + writeTsvBRow(readName, mateNum, result.role(), la); + } + } + + private static String mateNumColumn(final SAMRecord record) + { + if(!record.getReadPairedFlag()) + { + return "0"; + } + return record.getFirstOfPairFlag() ? "1" : "2"; + } + + private void writeTsvARow(final String readName, final String mateNum, final LiftBackResult result) throws IOException + { + mTsvAWriter.write(String.join(TSV_DELIM, + readName, + mateNum, + result.role().name(), + result.recordState().name(), + result.comp().name(), + result.outcome().name(), + result.decidingFeature() != null ? result.decidingFeature().name() : "", + String.valueOf(result.swapped()), + String.valueOf(result.inputMapq()), + String.valueOf(result.updatedMapq()), + String.valueOf(result.numXaAlts()), + String.valueOf(result.numRefAlts()), + String.valueOf(result.numTxAlts()), + String.valueOf(result.numLoci()), + String.valueOf(result.numDistinctCigarsAtPrimaryLocus()), + String.valueOf(result.txHasNCigar()), + String.valueOf(result.txSoftClipAtBoundary()), + String.valueOf(result.refSoftClipped()), + String.valueOf(result.refFullMatch()), + result.finalChrom(), + String.valueOf(result.finalPos()), + result.finalCigar(), + String.valueOf(result.hasNCigar()), + nullToEmpty(result.geneIds()), + nullToEmpty(result.notes()))); + mTsvAWriter.newLine(); + } + + private void writeTsvBRow( + final String readName, final String mateNum, final LiftBackResult.RecordRole role, final LiftedAlignment la) + throws IOException + { + mTsvBWriter.write(String.join(TSV_DELIM, + readName, + mateNum, + role.name(), + la.Source.name(), + la.OrigContig, + String.valueOf(la.OrigPos), + la.OrigCigar, + la.LiftedChrom, + String.valueOf(la.LiftedPos), + la.LiftedCigar, + String.valueOf(la.AlignmentScore), + String.valueOf(la.NumMismatches), + nullToEmpty(la.TransName), + nullToEmpty(la.GeneId), + nullToEmpty(la.GeneName))); + mTsvBWriter.newLine(); + } + + private static void writeHeader(final BufferedWriter writer, final String[] headers) throws IOException + { + writer.write(String.join(TSV_DELIM, headers)); + writer.newLine(); + } + + private static String nullToEmpty(final String value) + { + return value != null ? value : ""; + } + + @Override + public void close() throws IOException + { + mTsvAWriter.close(); + mTsvBWriter.close(); + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/LiftedAlignment.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/LiftedAlignment.java new file mode 100644 index 00000000000..b3a13360659 --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/LiftedAlignment.java @@ -0,0 +1,176 @@ +package com.hartwig.hmftools.tars.liftback; + +import com.hartwig.hmftools.tars.common.TarsConstants; + +// One component of a record's alignment set: the record itself or one of its XA alts, post-lift. +// TransName/GeneId/GeneName are set only for Tx-contig alignments. SoftClipAtBoundary is true only +// when the leading/trailing S abuts an interior exon boundary (not the contig's outermost edge). +public class LiftedAlignment +{ + public enum AlignmentSource + { + SELF, + XA_INPUT + } + + public final AlignmentSource Source; + public final String OrigContig; + public final int OrigPos; + public final String OrigCigar; + public final String LiftedChrom; + public final int LiftedPos; + public final String LiftedCigar; + public final int AlignmentScore; + public final int NumMismatches; + public final String TransName; + public final String GeneId; + public final String GeneName; + public final boolean SoftClipAtBoundary; + public final boolean ForwardStrand; + // +1 forward / -1 reverse for tx-contig alignments; 0 otherwise (no transcript strand known). + public final int TranscriptStrand; + + // Set by LiftBackResolver on the losing side of a discriminator decision. Kept for TSV-B diagnostics + // but excluded from the BAM XA tag. + public boolean Dropped = false; + + // Marks the alignment chosen as the BAM primary. Discriminator may flip it to an XA alt when bwa's + // primary pick was suboptimal (e.g. processed pseudogene over spliced parent). + public boolean IsPrimaryChoice = false; + + // bwa-mem-style score of the lifted placement against the genome, recomputed pre-discriminator so tx and ref + // candidates compare on one scale (bwa's AlignmentScore is tx-contig-space for a tx candidate, and absent for + // XA alts). Integer.MIN_VALUE = not computed; the discriminator then falls back to seeded randomness on ties. + public int GenomicScore = Integer.MIN_VALUE; + + public LiftedAlignment( + final AlignmentSource source, final String origContig, final int origPos, final String origCigar, + final String liftedChrom, final int liftedPos, final String liftedCigar, + final int alignmentScore, final int numMismatches, + final String transName, final String geneId, final String geneName, + final boolean softClipAtBoundary, final boolean forwardStrand) + { + this(source, origContig, origPos, origCigar, liftedChrom, liftedPos, liftedCigar, + alignmentScore, numMismatches, transName, geneId, geneName, + softClipAtBoundary, forwardStrand, 0); + } + + public LiftedAlignment( + final AlignmentSource source, final String origContig, final int origPos, final String origCigar, + final String liftedChrom, final int liftedPos, final String liftedCigar, + final int alignmentScore, final int numMismatches, + final String transName, final String geneId, final String geneName, + final boolean softClipAtBoundary, final boolean forwardStrand, final int transcriptStrand) + { + Source = source; + OrigContig = origContig; + OrigPos = origPos; + OrigCigar = origCigar; + LiftedChrom = liftedChrom; + LiftedPos = liftedPos; + LiftedCigar = liftedCigar; + AlignmentScore = alignmentScore; + NumMismatches = numMismatches; + TransName = transName; + GeneId = geneId; + GeneName = geneName; + SoftClipAtBoundary = softClipAtBoundary; + ForwardStrand = forwardStrand; + TranscriptStrand = transcriptStrand; + } + + // Copy with a normalized lifted placement (collapse / tail-extend output). The boundary-softclip flag + // only survives if the new cigar still carries a softclip; a fully reclaimed tail is no longer clipped. + public LiftedAlignment withLiftedCigar(final int liftedPos, final String liftedCigar) + { + boolean stillSoftClipped = liftedCigar.indexOf('S') >= 0; + return new LiftedAlignment( + Source, OrigContig, OrigPos, OrigCigar, + LiftedChrom, liftedPos, liftedCigar, + AlignmentScore, NumMismatches, + TransName, GeneId, GeneName, + SoftClipAtBoundary && stillSoftClipped, ForwardStrand, TranscriptStrand); + } + + public boolean fromTxContig() + { + return TransName != null; + } + + public boolean cigarHasN() + { + return LiftedCigar != null && LiftedCigar.indexOf('N') >= 0; + } + + public boolean cigarHasSoftClip() + { + return LiftedCigar != null && LiftedCigar.indexOf('S') >= 0; + } + + // True iff at least one N is flanked by >= MIN_JUNCTION_ANCHOR M both sides (one strong junction trusts the read). + public boolean cigarHasRealNJunction() + { + if(LiftedCigar == null) + { + return false; + } + + int prevMLength = 0; + int currentNum = 0; + + for(int i = 0; i < LiftedCigar.length(); ++i) + { + char c = LiftedCigar.charAt(i); + if(c >= '0' && c <= '9') + { + currentNum = currentNum * 10 + (c - '0'); + continue; + } + + if(c == 'N') + { + int nextM = nextMLength(LiftedCigar, i + 1); + if(prevMLength >= TarsConstants.MIN_JUNCTION_ANCHOR && nextM >= TarsConstants.MIN_JUNCTION_ANCHOR) + { + return true; + } + prevMLength = 0; + } + else if(c == 'M' || c == '=' || c == 'X') + { + prevMLength = currentNum; + } + else + { + // S/I/D/H/P break the adjacent anchor + prevMLength = 0; + } + + currentNum = 0; + } + + return false; + } + + private static int nextMLength(final String cigar, final int startIndex) + { + int num = 0; + for(int i = startIndex; i < cigar.length(); ++i) + { + char c = cigar.charAt(i); + if(c >= '0' && c <= '9') + { + num = num * 10 + (c - '0'); + } + else if(c == 'M' || c == '=' || c == 'X') + { + return num; + } + else + { + return 0; + } + } + return 0; + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/LiftedCoords.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/LiftedCoords.java new file mode 100644 index 00000000000..c054da3d3ba --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/LiftedCoords.java @@ -0,0 +1,9 @@ +package com.hartwig.hmftools.tars.liftback; + +// Translated genomic coordinates for a single alt-contig alignment. Used by SaTagRewriter (mapQ/NM pass through). +public record LiftedCoords( + String chromosome, + int position, + String cigarString) +{ +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/LiftedMateInfo.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/LiftedMateInfo.java new file mode 100644 index 00000000000..2ca1a95fe06 --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/LiftedMateInfo.java @@ -0,0 +1,21 @@ +package com.hartwig.hmftools.tars.liftback; + +// Lifted-coord summary for one primary alignment, cached in pass 1 to patch partner mate fields in pass 2. +// TODO: negativeStrand may need XS:A:+/- from gene strand for IGV colouring - revisit on redux integration. +public record LiftedMateInfo( + String chromosome, + int alignmentStart, + int alignmentEnd, + String liftedCigar, + boolean negativeStrand, + boolean unmapped) +{ + public static final LiftedMateInfo UNMAPPED = new LiftedMateInfo(null, 0, 0, null, false, true); + + public static LiftedMateInfo mapped( + final String chromosome, final int alignmentStart, final int alignmentEnd, final String liftedCigar, + final boolean negativeStrand) + { + return new LiftedMateInfo(chromosome, alignmentStart, alignmentEnd, liftedCigar, negativeStrand, false); + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/LiftedMateInfoCache.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/LiftedMateInfoCache.java new file mode 100644 index 00000000000..378c1f2d54e --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/LiftedMateInfoCache.java @@ -0,0 +1,61 @@ +package com.hartwig.hmftools.tars.liftback; + +import java.util.HashMap; +import java.util.Map; + +// Cache of lifted primary alignment info keyed by read name. Built in pass 1, consumed in pass 2 to patch +// mate fields (RNEXT/PNEXT/TLEN). Supplementary/secondary records are not cached - they don't populate mates. +public class LiftedMateInfoCache +{ + private final Map mLiftedMateInfoByReadName; + + public LiftedMateInfoCache() + { + mLiftedMateInfoByReadName = new HashMap<>(); + } + + public void recordPrimaryAlignment(final String readName, final boolean firstOfPair, final LiftedMateInfo info) + { + ReadPairLiftedMateInfo pairInfo = mLiftedMateInfoByReadName.computeIfAbsent(readName, k -> new ReadPairLiftedMateInfo()); + if(firstOfPair) + { + pairInfo.FirstInPair = info; + } + else + { + pairInfo.SecondInPair = info; + } + } + + public LiftedMateInfo getPartnerMateInfo(final String readName, final boolean firstOfPair) + { + ReadPairLiftedMateInfo pairInfo = mLiftedMateInfoByReadName.get(readName); + if(pairInfo == null) + { + return null; + } + return firstOfPair ? pairInfo.SecondInPair : pairInfo.FirstInPair; + } + + // used to mirror primary coords onto a supplementary whose own lift failed + public LiftedMateInfo getOwnPrimary(final String readName, final boolean firstOfPair) + { + ReadPairLiftedMateInfo pairInfo = mLiftedMateInfoByReadName.get(readName); + if(pairInfo == null) + { + return null; + } + return firstOfPair ? pairInfo.FirstInPair : pairInfo.SecondInPair; + } + + public int size() + { + return mLiftedMateInfoByReadName.size(); + } + + private static class ReadPairLiftedMateInfo + { + public LiftedMateInfo FirstInPair; + public LiftedMateInfo SecondInPair; + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/MateFieldPatcher.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/MateFieldPatcher.java new file mode 100644 index 00000000000..1ed05ff11ff --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/MateFieldPatcher.java @@ -0,0 +1,81 @@ +package com.hartwig.hmftools.tars.liftback; + +import static com.hartwig.hmftools.common.bam.SamRecordUtils.MATE_CIGAR_ATTRIBUTE; + +import htsjdk.samtools.SAMRecord; + +public class MateFieldPatcher +{ + public static void patchMateFields(final SAMRecord record, final LiftedMateInfoCache liftedMateInfoCache) + { + if(!record.getReadPairedFlag()) + { + return; + } + + LiftedMateInfo partnerInfo = liftedMateInfoCache.getPartnerMateInfo( + record.getReadName(), record.getFirstOfPairFlag()); + + if(partnerInfo == null || partnerInfo.unmapped()) + { + applyUnmappedPartner(record); + return; + } + + applyMappedPartner(record, partnerInfo); + } + + private static void applyUnmappedPartner(final SAMRecord record) + { + if(record.getReadUnmappedFlag()) + { + record.setReferenceName(SAMRecord.NO_ALIGNMENT_REFERENCE_NAME); + record.setAlignmentStart(SAMRecord.NO_ALIGNMENT_START); + } + record.setMateUnmappedFlag(true); + record.setMateReferenceName(record.getReferenceName()); + record.setMateAlignmentStart(record.getAlignmentStart()); + record.setMateNegativeStrandFlag(false); + record.setInferredInsertSize(0); + record.setProperPairFlag(false); + record.setAttribute(MATE_CIGAR_ATTRIBUTE, null); + } + + private static void applyMappedPartner(final SAMRecord record, final LiftedMateInfo partnerInfo) + { + record.setMateUnmappedFlag(false); + record.setMateReferenceName(partnerInfo.chromosome()); + record.setMateAlignmentStart(partnerInfo.alignmentStart()); + record.setMateNegativeStrandFlag(partnerInfo.negativeStrand()); + record.setAttribute(MATE_CIGAR_ATTRIBUTE, partnerInfo.liftedCigar()); + + if(record.getReadUnmappedFlag()) + { + record.setReferenceName(partnerInfo.chromosome()); + record.setAlignmentStart(partnerInfo.alignmentStart()); + record.setInferredInsertSize(0); + record.setProperPairFlag(false); + return; + } + + if(!record.getReferenceName().equals(partnerInfo.chromosome())) + { + record.setInferredInsertSize(0); + record.setProperPairFlag(false); + return; + } + + record.setInferredInsertSize(computeInferredInsertSize(record, partnerInfo)); + } + + // TLEN: signed distance between mates' 5' ends (strand-aware), +/-1 so the leftmost-5' mate is positive. + // Using 5' ends (not alignment-start) fixes sign for same-start pairs and magnitude when softclips extend an end. + static int computeInferredInsertSize(final SAMRecord record, final LiftedMateInfo partnerInfo) + { + int readFivePrime = record.getReadNegativeStrandFlag() ? record.getAlignmentEnd() : record.getAlignmentStart(); + int mateFivePrime = partnerInfo.negativeStrand() ? partnerInfo.alignmentEnd() : partnerInfo.alignmentStart(); + + int adjustment = mateFivePrime >= readFivePrime ? 1 : -1; + return mateFivePrime - readFivePrime + adjustment; + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/Outcome.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/Outcome.java new file mode 100644 index 00000000000..77a30e51eee --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/Outcome.java @@ -0,0 +1,10 @@ +package com.hartwig.hmftools.tars.liftback; + +// Which contig source the final primary came from. The discriminator's decision, reduced to three +// values; the reason rides alongside in DecidingFeature and the locus count in LiftBackResult.numLoci. +public enum Outcome +{ + REF, + TX, + UNRESOLVED +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/RecordState.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/RecordState.java new file mode 100644 index 00000000000..f53d94d5639 --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/RecordState.java @@ -0,0 +1,12 @@ +package com.hartwig.hmftools.tars.liftback; + +// Whether a record reached the discriminator at all. RESOLVED means the Outcome/DecidingFeature pair +// is meaningful; the other three are record states that bypass the contest entirely and drive how the +// SAM record is rewritten (see LiftBackRecordOps). +public enum RecordState +{ + RESOLVED, + UNMAPPED, + LIFT_FAILED, + SUPPLEMENTARY +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/SaTagRewriter.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/SaTagRewriter.java new file mode 100644 index 00000000000..badaecfe874 --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/SaTagRewriter.java @@ -0,0 +1,61 @@ +package com.hartwig.hmftools.tars.liftback; + +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import com.hartwig.hmftools.common.bam.SupplementaryReadData; + +// Rewrites each SA tag entry from tx-contig to genomic coordinates. +// NM is not recomputed - it disagrees with the rewritten CIGAR once N operators are introduced, same as the read's own NM. +// Entries that fail to lift are dropped; survivors are deduped by (chrom, pos, strand, CIGAR). +public class SaTagRewriter +{ + public static String rewriteSaTag(final String saTagValue, final LiftBackResolver resolver) + { + return rewriteSaTag(saTagValue, resolver, Collections.emptySet()); + } + + // excludeKeys: lifted entry keys (chrom:pos:strand:cigar, clips normalised H->S) of supplementaries dropped from + // the output (excluded region / orphan / low-AS). Their SA entries are removed so the primary never references a + // supp that isn't emitted. Clips are normalised because the dropped supp RECORD is hard-clipped while its SA-tag + // entry here is soft-clipped - see LiftBackGroupProcessor.suppSaKey. + public static String rewriteSaTag(final String saTagValue, final LiftBackResolver resolver, final Set excludeKeys) + { + if(saTagValue == null || saTagValue.isEmpty()) + { + return null; + } + + List alignments = SupplementaryReadData.extractAlignments(saTagValue); + if(alignments == null) + { + return null; + } + + StringBuilder rewritten = new StringBuilder(); + Set seenEntryKeys = new HashSet<>(); + + for(SupplementaryReadData alignment : alignments) + { + LiftedCoords lifted = resolver.liftCoords(alignment.Chromosome, alignment.Position, alignment.Cigar); + if(lifted == null) + continue; + + String entryKey = lifted.chromosome() + ':' + lifted.position() + ':' + alignment.Strand + ':' + + lifted.cigarString().replace('H', 'S'); + if(excludeKeys.contains(entryKey)) + continue; + if(!seenEntryKeys.add(entryKey)) + continue; + + SupplementaryReadData liftedData = new SupplementaryReadData( + lifted.chromosome(), lifted.position(), alignment.Strand, lifted.cigarString(), + alignment.MapQuality, alignment.NM); + rewritten.append(liftedData.asSamTag()).append(';'); + } + + return rewritten.length() == 0 ? null : rewritten.toString(); + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/ShardedChunkProducer.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/ShardedChunkProducer.java new file mode 100644 index 00000000000..8e1cf7b095d --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/ShardedChunkProducer.java @@ -0,0 +1,238 @@ +package com.hartwig.hmftools.tars.liftback; + +import static com.hartwig.hmftools.tars.common.TarsConfig.TARS_LOGGER; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.LongAdder; + +import com.hartwig.hmftools.tars.liftback.shard.BamShardSplitter; +import com.hartwig.hmftools.tars.liftback.shard.ShardRecordIterator; + +import htsjdk.samtools.SAMFileHeader; +import htsjdk.samtools.SAMRecord; +import htsjdk.samtools.SamReader; +import htsjdk.samtools.SamReaderFactory; +import htsjdk.samtools.ValidationStringency; + +// Reads the input across several threads, one per byte range from BamShardSplitter. A single-thread BGZF parse +// can't keep the workers fed; the ranges split on read-name boundaries so each shard still emits whole fragments. +// A daemon thread logs aggregate progress; END_OF_STREAM is queued per worker once all shards drain. +public class ShardedChunkProducer extends Thread +{ + private final String mInputBam; + private final String mRefGenomeFile; + private final BlockingQueue> mQueue; + private final int mWorkerCount; + private final int mChunkTargetReads; + private final int mShardCount; + + private static final long PROGRESS_INTERVAL_MS = 15_000; + private static final int PROGRESS_BAR_WIDTH = 24; + + public ShardedChunkProducer( + final String inputBam, final String refGenomeFile, final BlockingQueue> queue, + final int workerCount, final int chunkTargetReads, final int shardCount) + { + mInputBam = inputBam; + mRefGenomeFile = refGenomeFile; + mQueue = queue; + mWorkerCount = workerCount; + mChunkTargetReads = chunkTargetReads; + mShardCount = Math.max(1, shardCount); + } + + @Override + public void run() + { + try + { + File bam = new File(mInputBam); + SAMFileHeader header = readHeader(bam); + List ranges = BamShardSplitter.computeSplits(bam, header, mShardCount); + TARS_LOGGER.info("liftback reading input across {} shard(s)", ranges.size()); + + // kept open until all shards finish so the monitor can read their offsets. + List iterators = new ArrayList<>(); + for(final BamShardSplitter.ShardRange range : ranges) + { + iterators.add(new ShardRecordIterator(bam, header, range)); + } + + LongAdder readsCounter = new LongAdder(); + AtomicBoolean done = new AtomicBoolean(false); + AtomicBoolean failed = new AtomicBoolean(false); + + Thread monitor = new Thread(() -> runMonitor(iterators, bam.length(), readsCounter, done), "tars-progress"); + monitor.setDaemon(true); + monitor.start(); + + List shardThreads = new ArrayList<>(); + for(final ShardRecordIterator iter : iterators) + { + Thread shard = new Thread(() -> readShard(iter, readsCounter, failed), "tars-shard"); + shard.start(); + shardThreads.add(shard); + } + + for(final Thread shard : shardThreads) + { + shard.join(); + } + + done.set(true); + monitor.interrupt(); + closeQuietly(iterators); + + if(failed.get()) + { + throw new IllegalStateException("a shard reader failed"); + } + + for(int i = 0; i < mWorkerCount; ++i) + { + mQueue.put(ChunkProducer.END_OF_STREAM); + } + } + catch(InterruptedException e) + { + Thread.currentThread().interrupt(); + TARS_LOGGER.error("liftback sharded producer interrupted"); + System.exit(1); + } + catch(Exception e) + { + TARS_LOGGER.error("liftback sharded producer failed: {}", e.toString()); + System.exit(1); + } + } + + private void readShard(final ShardRecordIterator iter, final LongAdder readsCounter, final AtomicBoolean failed) + { + try + { + ChunkProducer.streamChunks(iter, mChunkTargetReads, mQueue::put, readsCounter); + } + catch(InterruptedException e) + { + Thread.currentThread().interrupt(); + failed.set(true); + } + catch(Exception e) + { + TARS_LOGGER.error("liftback shard reader failed: {}", e.toString()); + failed.set(true); + } + } + + // periodic progress: a bar of compressed bytes consumed across the shards, with read count and throughput. + private void runMonitor( + final List iterators, final long fileLength, final LongAdder readsCounter, + final AtomicBoolean done) + { + long lastReads = 0; + long lastNanos = System.nanoTime(); + while(!done.get()) + { + try + { + Thread.sleep(PROGRESS_INTERVAL_MS); + } + catch(InterruptedException e) + { + return; + } + if(done.get()) + { + return; + } + + long consumed = 0; + for(final ShardRecordIterator iter : iterators) + { + consumed += iter.consumedBytes(); + } + + double percent = fileLength > 0 ? Math.min(100.0, consumed * 100.0 / fileLength) : 0.0; + long reads = readsCounter.sum(); + long nowNanos = System.nanoTime(); + double intervalSecs = (nowNanos - lastNanos) / 1e9; + long rate = intervalSecs > 0 ? (long) ((reads - lastReads) / intervalSecs) : 0; + + TARS_LOGGER.info("liftback {} {}% | {} reads | {}/s", + bar((int) percent), String.format("%.2f", percent), formatCount(reads), formatRate(rate)); + + lastReads = reads; + lastNanos = nowNanos; + } + } + + private static String bar(final int percent) + { + int filled = Math.min(PROGRESS_BAR_WIDTH, percent * PROGRESS_BAR_WIDTH / 100); + StringBuilder sb = new StringBuilder(PROGRESS_BAR_WIDTH + 2); + sb.append('['); + for(int i = 0; i < PROGRESS_BAR_WIDTH; ++i) + { + sb.append(i < filled ? '#' : '.'); + } + sb.append(']'); + return sb.toString(); + } + + // Rate stays in thousands (e.g. 1939K/s) rather than collapsing to whole M/s, which hides the actual throughput. + private static String formatRate(final long value) + { + if(value >= 1_000) + { + return String.format("%dK", value / 1_000); + } + return String.valueOf(value); + } + + private static String formatCount(final long value) + { + if(value >= 1_000_000_000) + { + return String.format("%.1fB", value / 1e9); + } + if(value >= 1_000_000) + { + return String.format("%dM", value / 1_000_000); + } + if(value >= 1_000) + { + return String.format("%dK", value / 1_000); + } + return String.valueOf(value); + } + + private static void closeQuietly(final List iterators) + { + for(final ShardRecordIterator iter : iterators) + { + try + { + iter.close(); + } + catch(Exception ignored) + { + } + } + } + + private SAMFileHeader readHeader(final File bam) throws IOException + { + try(SamReader reader = SamReaderFactory.makeDefault() + .validationStringency(ValidationStringency.SILENT) + .referenceSequence(new File(mRefGenomeFile)) + .open(bam)) + { + return reader.getFileHeader(); + } + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/SpliceLiftBackConfig.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/SpliceLiftBackConfig.java new file mode 100644 index 00000000000..df23cc5a4fc --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/SpliceLiftBackConfig.java @@ -0,0 +1,164 @@ +package com.hartwig.hmftools.tars.liftback; + +import static com.hartwig.hmftools.common.bamops.BamToolName.BAMTOOL_PATH; +import static com.hartwig.hmftools.common.genome.refgenome.RefGenomeSource.REF_GENOME; +import static com.hartwig.hmftools.common.genome.refgenome.RefGenomeSource.addRefGenomeConfig; +import static com.hartwig.hmftools.common.perf.TaskExecutor.addThreadOptions; +import static com.hartwig.hmftools.common.perf.TaskExecutor.parseThreads; +import static com.hartwig.hmftools.common.utils.config.CommonConfig.SAMPLE; +import static com.hartwig.hmftools.common.utils.config.CommonConfig.SAMPLE_DESC; +import static com.hartwig.hmftools.common.utils.config.ConfigUtils.addLoggingOptions; +import static com.hartwig.hmftools.common.utils.file.FileDelimiters.BAM_EXTENSION; +import static com.hartwig.hmftools.common.utils.file.FileDelimiters.TSV_EXTENSION; +import static com.hartwig.hmftools.common.utils.file.FileWriterUtils.OUTPUT_ID; +import static com.hartwig.hmftools.common.utils.file.FileWriterUtils.addOutputOptions; +import static com.hartwig.hmftools.common.utils.file.FileWriterUtils.checkCreateOutputDir; +import static com.hartwig.hmftools.common.utils.file.FileWriterUtils.parseOutputDir; +import static com.hartwig.hmftools.tars.common.TarsConstants.FILE_ID; + +import com.hartwig.hmftools.common.bamops.BamToolName; +import com.hartwig.hmftools.common.genome.refgenome.RefGenomeVersion; +import com.hartwig.hmftools.common.utils.config.ConfigBuilder; +import com.hartwig.hmftools.tars.common.TarsConstants; +import com.hartwig.hmftools.tars.liftback.supplementary.SupplementaryConfig; + +public class SpliceLiftBackConfig +{ + public static final String INPUT_BAM = "input_bam"; + public static final String INPUT_BAM_DESC = "Input BAM aligned against ref + transcript-contig FASTA"; + + public static final String CONTIG_SIDECAR = "contig_sidecar"; + public static final String CONTIG_SIDECAR_DESC = "Contig sidecar TSV from SpliceFastaBuilder"; + + // supplementary resolve (merge supplementaries into a spliced primary) and softclip tail-extension both run + // always; the following expose their thresholds for tuning the FP/yield tradeoff. + public static final String SUPP_IMPLIED_MIN_INTRON_LENGTH = "supp_implied_min_intron_length"; + public static final String SUPP_IMPLIED_MAX_INTRON_LENGTH = "supp_implied_max_intron_length"; + + public static final String RNA_UNMAP_REGIONS = "rna_unmap_regions"; + public static final String RNA_UNMAP_REGIONS_DESC = + "Curated excluded regions (Chromosome/PosStart/PosEnd), e.g. RNA rRNA / 7SL / multi-map zones; " + + "a fragment with a primary in any region is dropped before lifting"; + + public static final String WRITE_LIFTBACK_TSV = "write_liftback_tsv"; + public static final String WRITE_LIFTBACK_TSV_DESC = + "Write per-record liftback debug TSVs (records + alignments). Off by default -- whole-sample TSVs are huge"; + + // output file-type tokens, e.g. .tars.summary.tsv + public static final String SUMMARY_FILE_TYPE = "summary"; + public static final String RECORDS_FILE_TYPE = "records"; + public static final String ALIGNMENTS_FILE_TYPE = "alignments"; + + public final String SampleId; + public final String InputBam; + public final String RefGenomeFile; + public final RefGenomeVersion RefGenVersion; + public final String ContigSidecarFile; + public final SupplementaryConfig Supplementary; + public final String RnaUnmapRegionsFile; + public final boolean WriteLiftbackTsv; + public final String OutputDir; + public final String OutputId; + public final String BamToolPath; + public final int Threads; + + public SpliceLiftBackConfig(final ConfigBuilder configBuilder) + { + SampleId = configBuilder.getValue(SAMPLE); + InputBam = configBuilder.getValue(INPUT_BAM); + RefGenomeFile = configBuilder.getValue(REF_GENOME); + RefGenVersion = RefGenomeVersion.from(configBuilder); + ContigSidecarFile = configBuilder.getValue(CONTIG_SIDECAR); + Supplementary = new SupplementaryConfig( + true, + configBuilder.getInteger(SUPP_IMPLIED_MIN_INTRON_LENGTH), + configBuilder.getInteger(SUPP_IMPLIED_MAX_INTRON_LENGTH), + TarsConstants.DEFAULT_MAX_SUPP_MERGES, + false, + TarsConstants.DEFAULT_SOFTCLIP_TOLERANCE, + TarsConstants.DEFAULT_MAX_BOUNDARY_SHIFT); + RnaUnmapRegionsFile = configBuilder.getValue(RNA_UNMAP_REGIONS); + WriteLiftbackTsv = configBuilder.hasFlag(WRITE_LIFTBACK_TSV); + OutputDir = parseOutputDir(configBuilder); + OutputId = configBuilder.getValue(OUTPUT_ID); + BamToolPath = configBuilder.getValue(BAMTOOL_PATH); + Threads = parseThreads(configBuilder); + + if(OutputDir == null) + { + throw new IllegalArgumentException("missing required config: output_dir"); + } + + if(!checkCreateOutputDir(OutputDir)) + { + throw new IllegalStateException("failed to create output directory: " + OutputDir); + } + } + + // common stem for every output: .tars[.], mirroring Redux. Also namespaces the + // per-worker shard intermediates so concurrent or repeated runs into one output dir do not clobber each other. + public String filePrefix() + { + String prefix = OutputDir + SampleId + "." + FILE_ID; + if(OutputId != null) + { + prefix += "." + OutputId; + } + return prefix; + } + + // .tars.[.].tsv + public String formFilename(final String fileType) + { + String filename = OutputDir + SampleId + "." + FILE_ID + "." + fileType; + if(OutputId != null) + { + filename += "." + OutputId; + } + return filename + TSV_EXTENSION; + } + + // .tars[.][.].bam + public String formBamFilename(final String stage) + { + String filename = OutputDir + SampleId + "." + FILE_ID; + if(OutputId != null) + { + filename += "." + OutputId; + } + if(stage != null) + { + filename += "." + stage; + } + return filename + BAM_EXTENSION; + } + + public String formUnsortedBam() { return formBamFilename("unsorted"); } + + public String formOutputBam() { return formBamFilename(null); } + + public String formTsvAFile() { return formFilename(RECORDS_FILE_TYPE); } + + public String formTsvBFile() { return formFilename(ALIGNMENTS_FILE_TYPE); } + + public String formSummaryFile() { return formFilename(SUMMARY_FILE_TYPE); } + + public static void addConfig(final ConfigBuilder configBuilder) + { + configBuilder.addConfigItem(SAMPLE, true, SAMPLE_DESC); + configBuilder.addPath(INPUT_BAM, true, INPUT_BAM_DESC); + addRefGenomeConfig(configBuilder, true); + configBuilder.addPath(CONTIG_SIDECAR, true, CONTIG_SIDECAR_DESC); + configBuilder.addInteger(SUPP_IMPLIED_MIN_INTRON_LENGTH, + "Supplementary resolve: min implied intron length", TarsConstants.DEFAULT_MIN_INTRON_LENGTH); + configBuilder.addInteger(SUPP_IMPLIED_MAX_INTRON_LENGTH, + "Supplementary resolve: max implied intron length", TarsConstants.DEFAULT_MAX_INTRON_LENGTH); + configBuilder.addPath(RNA_UNMAP_REGIONS, false, RNA_UNMAP_REGIONS_DESC); + configBuilder.addFlag(WRITE_LIFTBACK_TSV, WRITE_LIFTBACK_TSV_DESC); + BamToolName.addConfig(configBuilder); + + addOutputOptions(configBuilder); + addThreadOptions(configBuilder); + addLoggingOptions(configBuilder); + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/TarsApplication.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/TarsApplication.java new file mode 100644 index 00000000000..759923ce52b --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/TarsApplication.java @@ -0,0 +1,501 @@ +package com.hartwig.hmftools.tars.liftback; + +import static com.hartwig.hmftools.common.bamops.BamToolName.fromPath; +import static com.hartwig.hmftools.common.perf.PerformanceCounter.runTimeMinsStr; +import static com.hartwig.hmftools.common.perf.TaskExecutor.runThreadTasks; +import static com.hartwig.hmftools.tars.common.TarsConstants.ALT_CONTIG_SUFFIX; +import static com.hartwig.hmftools.tars.common.TarsConstants.APP_NAME; +import static com.hartwig.hmftools.tars.common.TarsConfig.TARS_LOGGER; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.EnumMap; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.stream.Collectors; + +import com.google.common.collect.Lists; +import com.hartwig.hmftools.common.bamops.BamOperations; +import com.hartwig.hmftools.common.bamops.BamToolName; +import com.hartwig.hmftools.common.utils.config.ConfigBuilder; +import com.hartwig.hmftools.common.utils.file.FileWriterUtils; +import com.hartwig.hmftools.tars.common.ContigEntry; +import com.hartwig.hmftools.tars.common.ContigSidecar; +import com.hartwig.hmftools.tars.common.TarsConstants; +import com.hartwig.hmftools.tars.liftback.supplementary.AnnotatedJunctionIndex; +import com.hartwig.hmftools.tars.liftback.supplementary.AnnotatedJunctionLoader; +import com.hartwig.hmftools.common.region.ChrBaseRegion; +import com.hartwig.hmftools.tars.liftback.supplementary.SupplementaryRejectReason; +import com.hartwig.hmftools.tars.liftback.supplementary.SupplementaryStatistics; +import com.hartwig.hmftools.tars.liftback.overhang.OverhangGateStatistics; + +import htsjdk.samtools.SAMFileHeader; +import htsjdk.samtools.SAMRecord; +import htsjdk.samtools.SAMSequenceDictionary; +import htsjdk.samtools.SAMSequenceRecord; +import htsjdk.samtools.SamReader; +import htsjdk.samtools.SamReaderFactory; + +// Standalone driver over the bwa-mem2 BAM aligned against ref + tx contigs. Consumes bwa's name-grouped +// output directly (mates + supplementaries already contiguous, FASTQ order): ChunkProducer streams it once +// and cuts whole-fragment chunks at name boundaries, N LiftBackWorkers lift + emit each chunk to its own +// BAM + (headerless) TSV shard. No input sort, no index, no fragment cache; only the lifted OUTPUT is +// sorted + indexed. The per-record transform lives in LiftBackGroupProcessor (one per worker, no shared state). +public class TarsApplication +{ + private final SpliceLiftBackConfig mConfig; + + // bwa's name-grouped reads are batched into chunks of ~this many reads before handing to a worker. + static final int CHUNK_TARGET_READS = 5000; + + // bounded in-flight chunk queue so memory scales with concurrency, not sample size (the OOM guard). + private static final int CHUNK_QUEUE_DEPTH_PER_THREAD = 2; + + // upper bound on parallel input-reader shards; a few parse threads outpace the workers, more just block. + private static final int READER_SHARD_CAP = 8; + + public TarsApplication(final ConfigBuilder configBuilder) + { + mConfig = new SpliceLiftBackConfig(configBuilder); + } + + public void run() + { + long startTimeMs = System.currentTimeMillis(); + + List contigEntries = ContigSidecar.read(mConfig.ContigSidecarFile); + TARS_LOGGER.info("loaded {} contig entries", contigEntries.size()); + + LiftBackResources resources = buildResources(contigEntries); + SAMFileHeader outputHeader = buildOutputHeader(); + + List shardBams = Lists.newArrayList(); + List tsvAShards = Lists.newArrayList(); + List tsvBShards = Lists.newArrayList(); + + List workers = runChunkStream(resources, outputHeader, shardBams, tsvAShards, tsvBShards); + if(workers == null) + { + throw new RuntimeException("liftback chunk stream failed"); + } + + mergeWorkerStats(workers); + + TARS_LOGGER.info("liftback processing complete, mins({}); concatenating + sorting shards", runTimeMinsStr(startTimeMs)); + + String unsortedBam = mConfig.formUnsortedBam(); + if(!concatenateShards(shardBams, unsortedBam)) + { + throw new RuntimeException("failed to concatenate liftback BAM shards"); + } + + if(!sortAndIndex(unsortedBam, mConfig.formOutputBam())) + { + throw new RuntimeException("failed to sort + index lifted BAM"); + } + + if(mConfig.WriteLiftbackTsv) + { + concatenateTsvShards(tsvAShards, LiftBackWriter.TSV_A_HEADER_LINE, mConfig.formTsvAFile()); + concatenateTsvShards(tsvBShards, LiftBackWriter.TSV_B_HEADER_LINE, mConfig.formTsvBFile()); + } + + cleanupIntermediates(unsortedBam, shardBams, tsvAShards, tsvBShards); + + TARS_LOGGER.info("TarsApplication complete, mins({})", runTimeMinsStr(startTimeMs)); + } + + // One producer streams bwa's name-grouped BAM, cutting whole-fragment chunks; N workers lift + emit each + // chunk to its own BAM + TSV shard. No input sort, no index, no fragment cache. + private List runChunkStream( + final LiftBackResources resources, final SAMFileHeader outputHeader, + final List shardBams, final List tsvAShards, final List tsvBShards) + { + int workerCount = Math.max(mConfig.Threads, 1); + BlockingQueue> chunkQueue = + new ArrayBlockingQueue<>(Math.max(workerCount * CHUNK_QUEUE_DEPTH_PER_THREAD, 2)); + + // a single-thread BGZF parse starves the workers, so the input is read by several shard threads that + // each parse their own byte range (split on read-name-group boundaries). A handful saturates the queue; + // beyond that the bounded queue just blocks them, so the count is capped well below the worker pool. + int shardCount = Math.max(1, Math.min(workerCount, READER_SHARD_CAP)); + ShardedChunkProducer producer = new ShardedChunkProducer( + mConfig.InputBam, mConfig.RefGenomeFile, chunkQueue, workerCount, CHUNK_TARGET_READS, shardCount); + + List workers = Lists.newArrayList(); + List threadTasks = Lists.newArrayList(); + threadTasks.add(producer); + + for(int i = 0; i < workerCount; ++i) + { + String shardBam = formShardBamPath(i); + shardBams.add(shardBam); + + // null shard paths disable the worker's TSV writer; debug TSVs are opt-in (whole-sample is huge). + String tsvAShard = mConfig.WriteLiftbackTsv ? formTsvShardPath(i, "records") : null; + String tsvBShard = mConfig.WriteLiftbackTsv ? formTsvShardPath(i, "alignments") : null; + if(mConfig.WriteLiftbackTsv) + { + tsvAShards.add(tsvAShard); + tsvBShards.add(tsvBShard); + } + + LiftBackWorker worker = new LiftBackWorker(chunkQueue, resources, outputHeader, shardBam, tsvAShard, tsvBShard); + workers.add(worker); + threadTasks.add(worker); + } + + return runThreadTasks(threadTasks) ? workers : null; + } + + private LiftBackResources buildResources(final List contigEntries) + { + // exon + junction annotation are derived from the sidecar's exonSpans (all rows, including annotation-only + // ones for transcripts without a contig), so liftback needs no ensembl cache. + ExonRegionIndex exonIndex = ExonRegionIndex.fromContigEntries(contigEntries); + TARS_LOGGER.info("built annotated-exon index from sidecar"); + + // annotation-only rows have no contig to lift against, so the resolver sees only real contig entries. + List liftEntries = contigEntries.stream() + .filter(entry -> !entry.isAnnotationOnly()).collect(Collectors.toList()); + LiftBackResolver resolver = new LiftBackResolver(liftEntries, exonIndex); + validateBamAgainstSidecar(resolver.contigNames()); + + Set annotated = AnnotatedJunctionLoader.fromContigEntries(contigEntries); + AnnotatedJunctionIndex junctionIndex = new AnnotatedJunctionIndex(annotated); + TARS_LOGGER.info("built {} annotated junctions from sidecar", annotated.size()); + + ExcludedRegions excludedRegions = null; + if(mConfig.RnaUnmapRegionsFile != null) + { + excludedRegions = ExcludedRegions.load(mConfig.RnaUnmapRegionsFile); + TARS_LOGGER.info("loaded excluded regions from {}", mConfig.RnaUnmapRegionsFile); + } + + return new LiftBackResources( + resolver, junctionIndex, mConfig.RefGenomeFile, + mConfig.Supplementary, excludedRegions); + } + + // fails fast on a BAM/sidecar mismatch -- root cause of a prior silent failure where alt contigs missing + // from the sidecar fell through as ref pass-through and leaked _tx names into mate fields. + private void validateBamAgainstSidecar(final Set sidecarContigs) + { + Set missing = new TreeSet<>(); + try(SamReader reader = SamReaderFactory.makeDefault() + .referenceSequence(new File(mConfig.RefGenomeFile)) + .open(new File(mConfig.InputBam))) + { + for(final SAMSequenceRecord sq : reader.getFileHeader().getSequenceDictionary().getSequences()) + { + String name = sq.getSequenceName(); + if(name.endsWith(ALT_CONTIG_SUFFIX) && !sidecarContigs.contains(name)) + { + missing.add(name); + } + } + } + catch(IOException e) + { + throw new RuntimeException("failed to read BAM header: " + mConfig.InputBam, e); + } + + if(!missing.isEmpty()) + { + throw new IllegalStateException(String.format( + "BAM/sidecar mismatch: %d alt contig(s) in BAM @SQ are absent from sidecar %s -- first few: %s", + missing.size(), mConfig.ContigSidecarFile, + missing.stream().limit(5).collect(Collectors.joining(",")))); + } + } + + // clone the input BAM header, mark it unsorted, and strip the _tx alt contigs from @SQ so the lifted BAM + // carries a pure genomic dictionary. + private SAMFileHeader buildOutputHeader() + { + try(SamReader reader = SamReaderFactory.makeDefault() + .referenceSequence(new File(mConfig.RefGenomeFile)) + .open(new File(mConfig.InputBam))) + { + SAMFileHeader header = reader.getFileHeader().clone(); + header.setSortOrder(SAMFileHeader.SortOrder.unsorted); + + SAMSequenceDictionary dict = header.getSequenceDictionary(); + List kept = new ArrayList<>(); + int dropped = 0; + for(final SAMSequenceRecord seq : dict.getSequences()) + { + if(seq.getSequenceName().endsWith(ALT_CONTIG_SUFFIX)) + { + ++dropped; + } + else + { + kept.add(seq); + } + } + header.setSequenceDictionary(new SAMSequenceDictionary(kept)); + TARS_LOGGER.info("dropped {} alt contig @SQ entries from output header", dropped); + return header; + } + catch(IOException e) + { + throw new RuntimeException("failed to read header from " + mConfig.InputBam, e); + } + } + + private void mergeWorkerStats(final List workers) + { + LiftBackStats combined = new LiftBackStats(); + for(final LiftBackWorker worker : workers) + { + combined.merge(worker.liftBackStats()); + } + + // Aggregate per-pass effectiveness so the always-written summary carries it (no opt-in TSV needed). + combined.setPassEffects(aggregatePassEffects(workers)); + + combined.logSummary(); + + try + { + combined.writeSummary(mConfig.formSummaryFile()); + } + catch(IOException e) + { + TARS_LOGGER.warn("failed to write liftback summary: {}", e.toString()); + } + + logSupplementaryStats(workers); + logOverhangStats(workers); + + long collapsedLeading = 0; + long collapsedTrailing = 0; + long overCapUnmapped = 0; + long excludedReads = 0; + for(final LiftBackWorker worker : workers) + { + OverhangGateStatistics overhang = worker.overhangStatistics(); + if(overhang != null) + { + collapsedLeading += overhang.collapsedLeading(); + collapsedTrailing += overhang.collapsedTrailing(); + } + overCapUnmapped += worker.overCapUnmapped(); + excludedReads += worker.excludedReads(); + } + if(collapsedLeading > 0 || collapsedTrailing > 0) + { + TARS_LOGGER.info("overhang-gate collapse: leading={} trailing={}", collapsedLeading, collapsedTrailing); + } + if(overCapUnmapped > 0) + { + TARS_LOGGER.info("over-cap unmap: {} primaries unmapped (MAPQ 0 + no XA, maps past the bwa XA cap)", overCapUnmapped); + } + if(excludedReads > 0) + { + TARS_LOGGER.info("excluded-region reads (primary unmapped / supp dropped, post-lift): {}", excludedReads); + } + } + + private static LiftBackStats.PassEffects aggregatePassEffects(final List workers) + { + int suppCandidates = 0, suppMerged = 0, suppClamped = 0; + long collapseLeading = 0, collapseTrailing = 0; + long reclaimRecords = 0, reclaimBasesLead = 0, reclaimBasesTrail = 0, altsDropped = 0; + long overCapUnmapped = 0, excludedReads = 0; + for(final LiftBackWorker worker : workers) + { + SupplementaryStatistics supplementary = worker.supplementaryStatistics(); + if(supplementary != null) + { + suppCandidates += supplementary.candidatesEvaluated(); + suppMerged += supplementary.mergedTotal(); + suppClamped += supplementary.suppClampApplied(); + } + OverhangGateStatistics overhang = worker.overhangStatistics(); + if(overhang != null) + { + collapseLeading += overhang.collapsedLeading(); + collapseTrailing += overhang.collapsedTrailing(); + reclaimRecords += overhang.recordsReclaimed(); + reclaimBasesLead += overhang.basesReclaimedLead(); + reclaimBasesTrail += overhang.basesReclaimedTrail(); + altsDropped += overhang.altsDropped(); + } + overCapUnmapped += worker.overCapUnmapped(); + excludedReads += worker.excludedReads(); + } + return new LiftBackStats.PassEffects( + suppCandidates, suppMerged, suppClamped, + collapseLeading, collapseTrailing, + reclaimRecords, reclaimBasesLead, reclaimBasesTrail, altsDropped, + overCapUnmapped, excludedReads); + } + + private static void logSupplementaryStats(final List workers) + { + int candidates = 0; + int merged = 0; + int clamp = 0; + int[] depth = new int[5]; + EnumMap rejects = new EnumMap<>(SupplementaryRejectReason.class); + for(final LiftBackWorker worker : workers) + { + SupplementaryStatistics stats = worker.supplementaryStatistics(); + if(stats == null) + continue; + candidates += stats.candidatesEvaluated(); + merged += stats.mergedTotal(); + clamp += stats.suppClampApplied(); + for(int d = 1; d <= 4; ++d) + { + depth[d] += stats.mergedAtChainDepth(d); + } + for(final SupplementaryRejectReason reason : SupplementaryRejectReason.values()) + { + rejects.merge(reason, stats.rejectCount(reason), Integer::sum); + } + } + TARS_LOGGER.info("supplementary-resolve summary: candidates={} merged={} suppClamped={} (depth1={} depth2={} depth3={} depth4={})", + candidates, merged, clamp, depth[1], depth[2], depth[3], depth[4]); + for(final SupplementaryRejectReason reason : SupplementaryRejectReason.values()) + { + if(rejects.getOrDefault(reason, 0) > 0) + { + TARS_LOGGER.debug("supplementary-resolve reject {}: {}", reason, rejects.get(reason)); + } + } + } + + private static void logOverhangStats(final List workers) + { + long reclaimed = 0; + long basesLead = 0; + long basesTrail = 0; + long altsDropped = 0; + for(final LiftBackWorker worker : workers) + { + OverhangGateStatistics stats = worker.overhangStatistics(); + if(stats == null) + continue; + reclaimed += stats.recordsReclaimed(); + basesLead += stats.basesReclaimedLead(); + basesTrail += stats.basesReclaimedTrail(); + altsDropped += stats.altsDropped(); + } + TARS_LOGGER.info("overhang-gate reclaim summary: records={} basesLead={} basesTrail={}; altsDropped={}", + reclaimed, basesLead, basesTrail, altsDropped); + } + + private boolean concatenateShards(final List shardBams, final String unsortedBam) + { + BamToolName toolName = fromPath(mConfig.BamToolPath); + TARS_LOGGER.info("concatenating {} liftback shards via {}", shardBams.size(), toolName); + // samtools cat is a plain BGZF block concat and rejects -@, so pass threads=1. + return BamOperations.concatenateBams(toolName, mConfig.BamToolPath, unsortedBam, shardBams, 1); + } + + private boolean sortAndIndex(final String unsortedBam, final String sortedBam) + { + if(mConfig.BamToolPath == null) + { + TARS_LOGGER.info("no -{} configured; leaving unsorted BAM at {}", BamToolName.BAMTOOL_PATH, unsortedBam); + return false; + } + + BamToolName toolName = fromPath(mConfig.BamToolPath); + TARS_LOGGER.info("sorting BAM via {}: {}", toolName, sortedBam); + + if(!BamOperations.sortBam(toolName, mConfig.BamToolPath, unsortedBam, sortedBam, mConfig.Threads)) + { + return false; + } + + // sambamba sort indexes inline; only samtools needs the explicit index pass. + if(toolName == BamToolName.SAMTOOLS) + { + if(!BamOperations.indexBam(toolName, mConfig.BamToolPath, sortedBam, mConfig.Threads)) + { + return false; + } + } + + return true; + } + + // write the header line once, then append each worker's headerless data shard. + static void concatenateTsvShards(final List shards, final String headerLine, final String outPath) + { + try(BufferedWriter writer = FileWriterUtils.createBufferedWriter(outPath)) + { + writer.write(headerLine); + writer.newLine(); + for(final String shard : shards) + { + try(BufferedReader reader = Files.newBufferedReader(Paths.get(shard))) + { + String line; + while((line = reader.readLine()) != null) + { + writer.write(line); + writer.newLine(); + } + } + } + } + catch(IOException e) + { + throw new RuntimeException("failed to concatenate TSV shards into " + outPath, e); + } + } + + private String formShardBamPath(final int index) + { + return mConfig.filePrefix() + ".shard_" + index + ".bam"; + } + + private String formTsvShardPath(final int index, final String kind) + { + return mConfig.filePrefix() + "." + kind + ".shard_" + index + ".tsv"; + } + + private void cleanupIntermediates( + final String unsortedBam, final List shardBams, + final List tsvAShards, final List tsvBShards) + { + deleteQuietly(unsortedBam); + shardBams.forEach(TarsApplication::deleteQuietly); + tsvAShards.forEach(TarsApplication::deleteQuietly); + tsvBShards.forEach(TarsApplication::deleteQuietly); + } + + private static void deleteQuietly(final String path) + { + try + { + Files.deleteIfExists(Paths.get(path)); + } + catch(IOException e) + { + TARS_LOGGER.warn("could not delete intermediate {}: {}", path, e.toString()); + } + } + + public static void main(final String[] args) + { + ConfigBuilder configBuilder = new ConfigBuilder(APP_NAME); + SpliceLiftBackConfig.addConfig(configBuilder); + configBuilder.checkAndParseCommandLine(args); + + new TarsApplication(configBuilder).run(); + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/overhang/BoundaryReclaim.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/overhang/BoundaryReclaim.java new file mode 100644 index 00000000000..57fd23198fb --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/overhang/BoundaryReclaim.java @@ -0,0 +1,87 @@ +package com.hartwig.hmftools.tars.liftback.overhang; + +import com.hartwig.hmftools.tars.common.TarsConstants; + +// Re-evaluates a lifted terminal region (bwa's M-tail anchor + soft-clip) against the contiguous +// reference, independent of bwa's tx-contig M/S split. bwa scored the alignment over the spliced +// transcript contig, so a locally-negative stretch (e.g. a 3M that is 2 matches + 1 mismatch, scoring +// -2) survived as M because the surrounding alignment carried it. Lifted back to the genome that +// decision no longer holds, so the boundary is recomputed here: the reclaim length is the prefix that +// maximises cumulative bwa-mem score walking from the near-exon boundary outward. +public final class BoundaryReclaim +{ + private BoundaryReclaim() { } + + // read and ref are aligned index-for-index, ordered from the near-exon boundary outward. Returns the + // longest prefix in [0, len] reaching the highest cumulative score; 0 if no positive-scoring prefix. + // >= keeps the longest tie, so an internal mismatch followed by genuine matches reclaims those matches + // (a tie can only be reached by a match step, so the prefix never ends on a trailing mismatch). + public static int maxScoringPrefix(final byte[] read, final byte[] ref) + { + int len = Math.min(read.length, ref.length); + int score = 0; + int bestScore = 0; + int bestLength = 0; + for(int i = 0; i < len; ++i) + { + score += basesEqualIgnoreCase(read[i], ref[i]) ? TarsConstants.MATCH : TarsConstants.MISMATCH; + if(score >= bestScore && score > 0) + { + bestScore = score; + bestLength = i + 1; + } + } + return bestLength; + } + + // Highest cumulative score over any prefix [0, len] walking from the boundary outward (>= 0). This is the + // value maxScoringPrefix reaches - use it to compare a contiguous extension against an alternative placement. + public static int maxPrefixScore(final byte[] read, final byte[] ref) + { + int len = Math.min(read.length, ref.length); + int score = 0; + int bestScore = 0; + for(int i = 0; i < len; ++i) + { + score += basesEqualIgnoreCase(read[i], ref[i]) ? TarsConstants.MATCH : TarsConstants.MISMATCH; + if(score > bestScore) + { + bestScore = score; + } + } + return bestScore; + } + + // Straight bwa-mem score of read vs ref over their overlapping length (no clipping). Used to score an anchor + // against a fixed placement (e.g. bwa's far-exon position). + public static int score(final byte[] read, final byte[] ref) + { + int total = 0; + int len = Math.min(read.length, ref.length); + for(int i = 0; i < len; ++i) + { + total += basesEqualIgnoreCase(read[i], ref[i]) ? TarsConstants.MATCH : TarsConstants.MISMATCH; + } + return total; + } + + // Reverses a window so a region whose near-exon boundary sits at its end can be walked boundary-outward. + public static byte[] reversed(final byte[] bases) + { + byte[] out = new byte[bases.length]; + for(int i = 0; i < bases.length; ++i) + { + out[i] = bases[bases.length - 1 - i]; + } + return out; + } + + static boolean basesEqualIgnoreCase(final byte a, final byte b) + { + if(a == b) + { + return true; + } + return (a & ~0x20) == (b & ~0x20); + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/overhang/OverhangGate.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/overhang/OverhangGate.java new file mode 100644 index 00000000000..28a823f6f24 --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/overhang/OverhangGate.java @@ -0,0 +1,495 @@ +package com.hartwig.hmftools.tars.liftback.overhang; + +import static com.hartwig.hmftools.tars.common.TarsConstants.MIN_OVERHANG_LENGTH; +import static com.hartwig.hmftools.tars.common.TarsConstants.MIN_OVERHANG_SCORE; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import com.hartwig.hmftools.common.bam.CigarUtils; +import com.hartwig.hmftools.tars.liftback.supplementary.RefSequenceSource; + +import htsjdk.samtools.CigarElement; +import htsjdk.samtools.CigarOperator; + +// Gates a lifted alignment's terminal splice-junction anchors ("overhangs") against the reference genome, and +// reclaims genuine ref-matching bases bwa soft-clipped at read tails. +// +// peel() - iterative terminal collapse. For each end (trailing then leading) it walks the terminal shape +// "<...>M nN yM [eS]" (or its mirror) inward. It engages a terminal junction only when the trigger applies: +// an adjacent soft-clip (short overhang with a clip) OR the alignment carries more than one junction (bwa's +// multi-splice route may not weigh the intronic alternative). A single clean junction with no clip is left +// untouched. The anchor y is scored at its far-exon placement; the junction is kept iff that score exceeds +// MIN_OVERHANG_SCORE (aligning the anchor beats clipping it), otherwise the N is dropped, the y+e window is +// read through the intron contiguously reclaiming its best-scoring prefix to M, and the exposed inner junction +// becomes terminal and is re-gated. peel() reports dropped=true when it collapses a junction-bearing alignment +// to a purely contiguous one (no N remains): the caller drops such an XA alt from the tag, but never self. +// +// reclaimTerminalSoftClip() - the standalone reclaim. For any surviving terminal soft-clip it walks the clip into +// contiguous reference and converts the best-scoring prefix S -> M. The caller runs this post-supplementary-resolve on +// the chosen primary only when it is a tx-match, so a genomic/ref over-clip is left as bwa placed it and a clip +// supplementary resolve needs to splice is never consumed before it runs. +public class OverhangGate +{ + private final RefSequenceSource mRefSource; + private final OverhangGateStatistics mStatistics; + + public OverhangGate(final RefSequenceSource refSource) + { + mRefSource = refSource; + mStatistics = new OverhangGateStatistics(); + } + + public OverhangGateStatistics statistics() + { + return mStatistics; + } + + // The gate is a no-op without a ref-genome handle (each pass self-guards on it). + public boolean enabled() + { + return mRefSource != null; + } + + // Final (start, cigar) of a gate pass, plus whether an XA-alt placement should be dropped from the tag. + public record Result(int pos, String cigar, boolean dropped) + { + } + + // ============================ iterative terminal collapse ============================ + + public Result peel(final String chromosome, final int alignmentStart, final String cigar, final byte[] readBases) + { + if(mRefSource == null || chromosome == null || cigar == null || readBases == null || readBases.length == 0) + { + return new Result(alignmentStart, cigar, false); + } + + int pos = alignmentStart; + String working = cigar; + + // Trailing then leading; both ends peel inward, each collapse exposing the next inner junction. + while(working.indexOf('N') >= 0) + { + CollapseResult collapse = tryCollapseTrailing(chromosome, pos, working, readBases); + if(!collapse.collapsed()) + { + break; + } + working = collapse.cigar(); + mStatistics.countCollapsedTrailing(); + } + + while(working.indexOf('N') >= 0) + { + CollapseResult collapse = tryCollapseLeading(chromosome, pos, working, readBases); + if(!collapse.collapsed()) + { + break; + } + pos = collapse.pos(); + working = collapse.cigar(); + mStatistics.countCollapsedLeading(); + } + + // A junction-bearing alignment collapsed to a purely contiguous one has lost the splice that distinguished + // it; the caller drops such an XA alt (never self). + boolean dropped = cigar.indexOf('N') >= 0 && working.indexOf('N') < 0; + return new Result(pos, working, dropped); + } + + private CollapseResult tryCollapseTrailing( + final String chromosome, final int alignmentStart, final String cigar, final byte[] readBases) + { + List elements = CigarUtils.cigarElementsFromStr(cigar); + if(elements.size() < 3 || CigarUtils.hasHardClip(elements)) + { + return CollapseResult.unchanged(); + } + + int last = elements.size() - 1; + + // shape: "<...>M nN yM [eS]" + boolean hasSoftclip = elements.get(last).getOperator() == CigarOperator.S; + int e = hasSoftclip ? elements.get(last).getLength() : 0; + int anchorIndex = hasSoftclip ? last - 1 : last; + int intronIndex = anchorIndex - 1; + int nearIndex = intronIndex - 1; + if(nearIndex < 0) + { + return CollapseResult.unchanged(); + } + + CigarElement anchor = elements.get(anchorIndex); + if(!isMatch(anchor.getOperator())) + { + return CollapseResult.unchanged(); + } + if(elements.get(intronIndex).getOperator() != CigarOperator.N) + { + return CollapseResult.unchanged(); + } + CigarElement nearExon = elements.get(nearIndex); + if(!isMatch(nearExon.getOperator())) + { + return CollapseResult.unchanged(); + } + + // trigger: engage only with an adjacent soft-clip or more than one junction (Case 1 / Case 2). + if(e == 0 && countIntrons(elements) <= 1) + { + return CollapseResult.unchanged(); + } + + int y = anchor.getLength(); + // a long overhang is trusted outright (a real junction even with a couple of mismatches); only short overhangs + // are scored. See TarsConstants.MIN_OVERHANG_LENGTH. + if(y > MIN_OVERHANG_LENGTH) + { + return CollapseResult.unchanged(); + } + int window = y + e; + int intronLen = elements.get(intronIndex).getLength(); + // genomic end of the near exon, just before the intron + int nearEnd = alignmentStart + CigarUtils.cigarAlignedLength(elements.subList(0, intronIndex)) - 1; + byte[] ref = mRefSource.getBases(chromosome, nearEnd + 1, nearEnd + window); + if(ref == null || ref.length != window) + { + return CollapseResult.unchanged(); + } + + // Score the overhang at its far exon (the junction placement). A null far ref (off contig) can't be scored, + // so it never keeps the junction. Case 1 (soft clip) vs Case 2 (multiple junctions) differ in keepJunction. + byte[] anchorRead = Arrays.copyOfRange(readBases, readBases.length - window, readBases.length - window + y); + byte[] farRef = mRefSource.getBases(chromosome, nearEnd + intronLen + 1, nearEnd + intronLen + y); + int overhangScore = (farRef != null && farRef.length == y) ? BoundaryReclaim.score(anchorRead, farRef) : Integer.MIN_VALUE; + if(keepJunction(overhangScore, anchorRead, ref, e)) + { + return CollapseResult.unchanged(); + } + + // drop the intron; reclaim the contiguous-scoring prefix into the near exon, clip the remainder. + byte[] readWindow = Arrays.copyOfRange(readBases, readBases.length - window, readBases.length); + int reclaimed = BoundaryReclaim.maxScoringPrefix(readWindow, ref); + List merged = new ArrayList<>(elements.subList(0, nearIndex)); + merged.add(new CigarElement(nearExon.getLength() + reclaimed, CigarOperator.M)); + int residual = window - reclaimed; + if(residual > 0) + { + merged.add(new CigarElement(residual, CigarOperator.S)); + } + return new CollapseResult(true, alignmentStart, CigarUtils.cigarElementsToStr(merged)); + } + + private CollapseResult tryCollapseLeading( + final String chromosome, final int alignmentStart, final String cigar, final byte[] readBases) + { + List elements = CigarUtils.cigarElementsFromStr(cigar); + if(elements.size() < 3 || CigarUtils.hasHardClip(elements)) + { + return CollapseResult.unchanged(); + } + + // shape: "[eS] yM nN <...>M" + boolean hasSoftclip = elements.get(0).getOperator() == CigarOperator.S; + int e = hasSoftclip ? elements.get(0).getLength() : 0; + int anchorIndex = hasSoftclip ? 1 : 0; + int intronIndex = anchorIndex + 1; + int nearIndex = intronIndex + 1; + if(nearIndex >= elements.size()) + { + return CollapseResult.unchanged(); + } + + CigarElement anchor = elements.get(anchorIndex); + if(!isMatch(anchor.getOperator())) + { + return CollapseResult.unchanged(); + } + if(elements.get(intronIndex).getOperator() != CigarOperator.N) + { + return CollapseResult.unchanged(); + } + CigarElement nearExon = elements.get(nearIndex); + if(!isMatch(nearExon.getOperator())) + { + return CollapseResult.unchanged(); + } + + // trigger: engage only with an adjacent soft-clip or more than one junction (Case 1 / Case 2). + if(e == 0 && countIntrons(elements) <= 1) + { + return CollapseResult.unchanged(); + } + + int y = anchor.getLength(); + // a long overhang is trusted outright (a real junction even with a couple of mismatches); only short overhangs + // are scored. See TarsConstants.MIN_OVERHANG_LENGTH. + if(y > MIN_OVERHANG_LENGTH) + { + return CollapseResult.unchanged(); + } + int window = y + e; + int intron = elements.get(intronIndex).getLength(); + // leading softclip consumes no reference, so nearStart skips only anchor + intron. + int nearStart = alignmentStart + y + intron; + byte[] ref = mRefSource.getBases(chromosome, nearStart - window, nearStart - 1); + if(ref == null || ref.length != window) + { + return CollapseResult.unchanged(); + } + + // the near-exon boundary sits at the END of the leading window, so reverse both so the score walk runs + // boundary-outward. + byte[] readWindow = BoundaryReclaim.reversed(Arrays.copyOfRange(readBases, 0, window)); + byte[] nearRef = BoundaryReclaim.reversed(ref); + + // Score the overhang at its far exon (the junction placement); the leading anchor sits at [alignmentStart, +y). + // Score is order-independent, so no reversal needed. Case 1 (soft clip) vs Case 2 (multiple junctions) differ + // in keepJunction. + byte[] anchorRead = Arrays.copyOfRange(readBases, e, e + y); + byte[] farRef = mRefSource.getBases(chromosome, alignmentStart, alignmentStart + y - 1); + int overhangScore = (farRef != null && farRef.length == y) ? BoundaryReclaim.score(anchorRead, farRef) : Integer.MIN_VALUE; + if(keepJunction(overhangScore, anchorRead, ref, e)) + { + return CollapseResult.unchanged(); + } + + int reclaimed = BoundaryReclaim.maxScoringPrefix(readWindow, nearRef); + int residual = window - reclaimed; + List merged = new ArrayList<>(); + if(residual > 0) + { + merged.add(new CigarElement(residual, CigarOperator.S)); + } + merged.add(new CigarElement(reclaimed + nearExon.getLength(), CigarOperator.M)); + for(int i = nearIndex + 1; i < elements.size(); ++i) + { + merged.add(elements.get(i)); + } + return new CollapseResult(true, nearStart - reclaimed, CigarUtils.cigarElementsToStr(merged)); + } + + // Decide whether to keep a short overhang's junction, by case. Case 1 (adjacent soft clip, e > 0): keep the junction + // if the overhang's far-exon score beats the clip penalty (MIN_OVERHANG_SCORE), else collapse. Case 2 (more than one + // junction, no soft clip, e == 0): keep the junction if the overhang aligns positively (score > 0); only when it does + // not (score <= 0) is the intronic reference checked - the junction is collapsed only if reading the overhang + // contiguously on the reference scores strictly higher than the junction (far exon) placement, otherwise it is kept. + private static boolean keepJunction(final int overhangScore, final byte[] anchorRead, final byte[] windowRef, final int e) + { + if(e > 0) + { + return overhangScore > MIN_OVERHANG_SCORE; + } + if(overhangScore > 0) + { + return true; + } + byte[] contiguousRef = Arrays.copyOfRange(windowRef, 0, anchorRead.length); + int altRefScore = BoundaryReclaim.score(anchorRead, contiguousRef); + return altRefScore <= overhangScore; + } + + // Result of one collapse step. newStart shifts only on a leading collapse. + private record CollapseResult(boolean collapsed, int pos, String cigar) + { + static CollapseResult unchanged() + { + return new CollapseResult(false, 0, null); + } + } + + // ============================ standalone terminal softclip reclaim ============================ + + public Result reclaimTerminalSoftClip( + final String chromosome, final int alignmentStart, final String cigar, final byte[] readBases) + { + if(mRefSource == null || chromosome == null || cigar == null || readBases == null || readBases.length == 0) + { + return new Result(alignmentStart, cigar, false); + } + + List elements = CigarUtils.cigarElementsFromStr(cigar); + if(elements.isEmpty() || CigarUtils.hasHardClip(elements)) + { + return new Result(alignmentStart, cigar, false); + } + + List working = new ArrayList<>(elements); + int alignmentEnd = alignmentStart + CigarUtils.cigarAlignedLength(working) - 1; + + int trailReclaimed = reclaimSide(Side.TRAILING, chromosome, alignmentStart, alignmentEnd, readBases, working); + int leadReclaimed = reclaimSide(Side.LEADING, chromosome, alignmentStart, alignmentEnd, readBases, working); + + if(trailReclaimed == 0 && leadReclaimed == 0) + { + return new Result(alignmentStart, cigar, false); + } + + mStatistics.countReclaimed(leadReclaimed, trailReclaimed); + return new Result(alignmentStart - leadReclaimed, CigarUtils.cigarElementsToStr(working), false); + } + + private int reclaimSide( + final Side side, final String chromosome, final int alignmentStart, final int alignmentEnd, + final byte[] readBases, final List working) + { + if(working.size() < 2) + { + return 0; + } + + int softclipIndex = side.softclipIndex(working); + int matchedIndex = side.matchedIndex(working); + + CigarElement softclip = working.get(softclipIndex); + if(softclip.getOperator() != CigarOperator.S) + { + return 0; + } + if(!isMatchedOp(working.get(matchedIndex).getOperator())) + { + return 0; + } + + int boundary = side.refBoundary(alignmentStart, alignmentEnd); + + int budget = softclip.getLength(); + if(side == Side.LEADING && alignmentStart - budget < 1) + { + budget = alignmentStart - 1; + } + if(budget <= 0) + { + return 0; + } + + byte[] refBases = side.fetchRef(mRefSource, chromosome, boundary, budget); + if(refBases == null || refBases.length == 0) + { + return 0; + } + + int walkLength = Math.min(refBases.length, budget); + int reclaimed = side.walk(readBases, refBases, softclip.getLength(), walkLength); + if(reclaimed == 0) + { + return 0; + } + + applyReclaim(working, softclipIndex, matchedIndex, reclaimed); + return reclaimed; + } + + private static void applyReclaim( + final List working, final int softclipIndex, final int matchedIndex, final int reclaimed) + { + CigarElement softclip = working.get(softclipIndex); + CigarElement matched = working.get(matchedIndex); + + working.set(matchedIndex, new CigarElement(matched.getLength() + reclaimed, matched.getOperator())); + + if(softclip.getLength() - reclaimed == 0) + { + working.remove(softclipIndex); + } + else + { + working.set(softclipIndex, new CigarElement(softclip.getLength() - reclaimed, CigarOperator.S)); + } + } + + private static boolean isMatch(final CigarOperator op) + { + return op == CigarOperator.M || op == CigarOperator.EQ; + } + + private static boolean isMatchedOp(final CigarOperator op) + { + return op == CigarOperator.M || op == CigarOperator.EQ || op == CigarOperator.X; + } + + private static int countIntrons(final List elements) + { + int count = 0; + for(final CigarElement element : elements) + { + if(element.getOperator() == CigarOperator.N) + { + ++count; + } + } + return count; + } + + // Abstracts leading/trailing geometry so reclaimSide is direction-free. + private enum Side + { + LEADING + { + @Override + int softclipIndex(final List cigar) { return 0; } + + @Override + int matchedIndex(final List cigar) { return 1; } + + @Override + int refBoundary(final int alignmentStart, final int alignmentEnd) { return alignmentStart; } + + @Override + byte[] fetchRef(final RefSequenceSource src, final String chr, final int boundary, final int budget) + { + return src.getBases(chr, boundary - budget, boundary - 1); + } + + @Override + int walk(final byte[] readBases, final byte[] refBases, final int softclipLength, final int walkLength) + { + // leading softclip's M boundary is at its inner end, so reverse to walk boundary-outward + byte[] readWin = BoundaryReclaim.reversed( + Arrays.copyOfRange(readBases, softclipLength - walkLength, softclipLength)); + byte[] refWin = BoundaryReclaim.reversed( + Arrays.copyOfRange(refBases, refBases.length - walkLength, refBases.length)); + return BoundaryReclaim.maxScoringPrefix(readWin, refWin); + } + }, + TRAILING + { + @Override + int softclipIndex(final List cigar) { return cigar.size() - 1; } + + @Override + int matchedIndex(final List cigar) { return cigar.size() - 2; } + + @Override + int refBoundary(final int alignmentStart, final int alignmentEnd) { return alignmentEnd; } + + @Override + byte[] fetchRef(final RefSequenceSource src, final String chr, final int boundary, final int budget) + { + return src.getBases(chr, boundary + 1, boundary + budget); + } + + @Override + int walk(final byte[] readBases, final byte[] refBases, final int softclipLength, final int walkLength) + { + // trailing softclip's M boundary is at its start, already boundary-outward + int readStart = readBases.length - softclipLength; + byte[] readWin = Arrays.copyOfRange(readBases, readStart, readStart + walkLength); + byte[] refWin = Arrays.copyOfRange(refBases, 0, walkLength); + return BoundaryReclaim.maxScoringPrefix(readWin, refWin); + } + }; + + abstract int softclipIndex(List cigar); + + abstract int matchedIndex(List cigar); + + abstract int refBoundary(int alignmentStart, int alignmentEnd); + + abstract byte[] fetchRef(RefSequenceSource src, String chr, int boundary, int budget); + + abstract int walk(byte[] readBases, byte[] refBases, int softclipLength, int walkLength); + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/overhang/OverhangGateStatistics.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/overhang/OverhangGateStatistics.java new file mode 100644 index 00000000000..7163ea8f6d1 --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/overhang/OverhangGateStatistics.java @@ -0,0 +1,58 @@ +package com.hartwig.hmftools.tars.liftback.overhang; + +// Counters for the overhang gate; aggregated across workers and logged at end-of-run. Junctions collapsed +// (leading/trailing) come from the peel; reclaimed records/bases from the standalone softclip reclaim; alts +// dropped from XA-alt placements the peel collapsed to a contiguous alignment. +public class OverhangGateStatistics +{ + private long mCollapsedLeading; + private long mCollapsedTrailing; + private long mRecordsReclaimed; + private long mBasesReclaimedLead; + private long mBasesReclaimedTrail; + private long mAltsDropped; + + public void countCollapsedLeading() { ++mCollapsedLeading; } + + public void countCollapsedTrailing() { ++mCollapsedTrailing; } + + public void countReclaimed(final long leadBases, final long trailBases) + { + ++mRecordsReclaimed; + mBasesReclaimedLead += leadBases; + mBasesReclaimedTrail += trailBases; + } + + public void countAltDropped() { ++mAltsDropped; } + + public long collapsedLeading() { return mCollapsedLeading; } + + public long collapsedTrailing() { return mCollapsedTrailing; } + + public long recordsReclaimed() { return mRecordsReclaimed; } + + public long basesReclaimedLead() { return mBasesReclaimedLead; } + + public long basesReclaimedTrail() { return mBasesReclaimedTrail; } + + public long altsDropped() { return mAltsDropped; } + + // Snapshot / restore so a discarded provisional mate decision can be rolled back without double-counting + // (see LiftBackGroupProcessor.processNameGroup). + public long[] snapshot() + { + return new long[] { + mCollapsedLeading, mCollapsedTrailing, mRecordsReclaimed, + mBasesReclaimedLead, mBasesReclaimedTrail, mAltsDropped }; + } + + public void restore(final long[] snapshot) + { + mCollapsedLeading = snapshot[0]; + mCollapsedTrailing = snapshot[1]; + mRecordsReclaimed = snapshot[2]; + mBasesReclaimedLead = snapshot[3]; + mBasesReclaimedTrail = snapshot[4]; + mAltsDropped = snapshot[5]; + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/shard/BamShardSplitter.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/shard/BamShardSplitter.java new file mode 100644 index 00000000000..694da90bb98 --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/shard/BamShardSplitter.java @@ -0,0 +1,156 @@ +package com.hartwig.hmftools.tars.liftback.shard; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.util.ArrayList; +import java.util.List; + +import htsjdk.samtools.BAMRecordCodec; +import htsjdk.samtools.SAMFileHeader; +import htsjdk.samtools.SAMRecord; +import htsjdk.samtools.util.BinaryCodec; +import htsjdk.samtools.util.BlockCompressedInputStream; + +// Splits a name-grouped BAM into N byte ranges, each starting on a read-name boundary so a shard reads +// [start, end) with whole fragments and no cross-shard reconciliation. A split is a BGZF virtual pointer at a +// group's first record: from a target byte offset, find the next block (by its magic), decode until the name +// changes, and take that record's pointer. Shard 0 starts after the header; the last runs to EOF. +public final class BamShardSplitter +{ + private BamShardSplitter() { } + + // [startVptr, endVptr); endVptr == EOF means "read to end of file". + public record ShardRange(long startVptr, long endVptr) + { + } + + public static final long EOF = Long.MAX_VALUE; + + private static final byte[] BGZF_MAGIC = { 0x1f, (byte) 0x8b, 0x08, 0x04 }; + private static final int SCAN_BUFFER = 64 * 1024; + + public static List computeSplits(final File bam, final SAMFileHeader header, final int shardCount) + throws IOException + { + long firstRecordVptr = headerEndVptr(bam, header); + if(shardCount <= 1) + { + return List.of(new ShardRange(firstRecordVptr, EOF)); + } + + long compressedLength = bam.length(); + List splits = new ArrayList<>(); + splits.add(firstRecordVptr); + + long lastBlockOffset = firstRecordVptr >>> 16; + for(int k = 1; k < shardCount; ++k) + { + long targetOffset = compressedLength * k / shardCount; + if(targetOffset <= lastBlockOffset) + continue; // shards collapsing onto the same block: fewer, larger shards is fine + + long blockStart = findBlockStart(bam, targetOffset); + if(blockStart < 0) + break; // no further block boundary: the remaining file is one shard + + long splitVptr = firstGroupBoundaryVptr(bam, header, blockStart << 16); + if(splitVptr > splits.get(splits.size() - 1)) + { + splits.add(splitVptr); + lastBlockOffset = splitVptr >>> 16; + } + } + + List ranges = new ArrayList<>(); + for(int i = 0; i < splits.size(); ++i) + { + long end = i + 1 < splits.size() ? splits.get(i + 1) : EOF; + ranges.add(new ShardRange(splits.get(i), end)); + } + return ranges; + } + + // walk the BAM header binary on a BGZF stream and return the virtual pointer of the first alignment record. + private static long headerEndVptr(final File bam, final SAMFileHeader header) throws IOException + { + try(BlockCompressedInputStream stream = new BlockCompressedInputStream(bam)) + { + BinaryCodec codec = new BinaryCodec(stream); + codec.readBytes(new byte[4]); // "BAM\1" + int textLength = codec.readInt(); + codec.readBytes(new byte[textLength]); // header text + int refCount = codec.readInt(); + for(int i = 0; i < refCount; ++i) + { + int nameLength = codec.readInt(); + codec.readBytes(new byte[nameLength]); // reference name + codec.readInt(); // reference length + } + return stream.getFilePointer(); + } + } + + // scan raw bytes from targetOffset for the next valid BGZF block header, return its compressed offset (-1 none). + private static long findBlockStart(final File bam, final long targetOffset) throws IOException + { + try(RandomAccessFile raf = new RandomAccessFile(bam, "r")) + { + long length = raf.length(); + byte[] buffer = new byte[SCAN_BUFFER]; + long offset = targetOffset; + + while(offset < length - 18) + { + raf.seek(offset); + int read = raf.read(buffer); + if(read <= 0) + { + return -1; + } + + for(int i = 0; i + 18 <= read; ++i) + { + if(buffer[i] == BGZF_MAGIC[0] && buffer[i + 1] == BGZF_MAGIC[1] + && buffer[i + 2] == BGZF_MAGIC[2] && buffer[i + 3] == BGZF_MAGIC[3] + && buffer[i + 12] == 0x42 && buffer[i + 13] == 0x43) // 'BC' subfield id + return offset + i; + } + + offset += read - 17; // overlap so a header straddling the buffer edge is not missed + } + } + return -1; + } + + // decode records from blockVptr tracking names; return the virtual pointer of the first record whose name + // differs from the first record seen -- i.e. the start of the second group at/after this block. + private static long firstGroupBoundaryVptr(final File bam, final SAMFileHeader header, final long blockVptr) + throws IOException + { + try(BlockCompressedInputStream stream = new BlockCompressedInputStream(bam)) + { + stream.seek(blockVptr); + BAMRecordCodec codec = new BAMRecordCodec(header); + codec.setInputStream(stream); + + String firstName = null; + long recordVptr = stream.getFilePointer(); + SAMRecord record; + while((record = codec.decode()) != null) + { + if(firstName == null) + { + firstName = record.getReadName(); + } + else if(!record.getReadName().equals(firstName)) + { + return recordVptr; + } + + recordVptr = stream.getFilePointer(); + } + return EOF; // no further boundary before EOF + } + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/shard/ShardRecordIterator.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/shard/ShardRecordIterator.java new file mode 100644 index 00000000000..8e7692fb4a7 --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/shard/ShardRecordIterator.java @@ -0,0 +1,84 @@ +package com.hartwig.hmftools.tars.liftback.shard; + +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Iterator; +import java.util.NoSuchElementException; + +import htsjdk.samtools.BAMRecordCodec; +import htsjdk.samtools.SAMFileHeader; +import htsjdk.samtools.SAMRecord; +import htsjdk.samtools.util.BlockCompressedInputStream; + +// Decodes the BAM records in one shard range [startVptr, endVptr). Both ends are read-name-group boundaries +// (see BamShardSplitter), so stopping when the next record begins at/after endVptr leaves every group whole. +public class ShardRecordIterator implements Iterator, Closeable +{ + private final BlockCompressedInputStream mStream; + private final BAMRecordCodec mCodec; + private final long mEndVptr; + private final long mStartOffset; + private SAMRecord mNext; + + public ShardRecordIterator(final File bam, final SAMFileHeader header, final BamShardSplitter.ShardRange range) + { + try + { + mStream = new BlockCompressedInputStream(bam); + mStream.seek(range.startVptr()); + } + catch(IOException e) + { + throw new UncheckedIOException(e); + } + + mCodec = new BAMRecordCodec(header); + mCodec.setInputStream(mStream); + mEndVptr = range.endVptr(); + mStartOffset = range.startVptr() >>> 16; + mNext = readNext(); + } + + // compressed bytes this shard has consumed so far -- the progress monitor sums these across shards. + public long consumedBytes() + { + return Math.max(0, (mStream.getFilePointer() >>> 16) - mStartOffset); + } + + private SAMRecord readNext() + { + if(mEndVptr != BamShardSplitter.EOF && mStream.getFilePointer() >= mEndVptr) + { + return null; + } + + return mCodec.decode(); // null at EOF + } + + @Override + public boolean hasNext() + { + return mNext != null; + } + + @Override + public SAMRecord next() + { + if(mNext == null) + { + throw new NoSuchElementException(); + } + + SAMRecord record = mNext; + mNext = readNext(); + return record; + } + + @Override + public void close() throws IOException + { + mStream.close(); + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/supplementary/AnnotatedJunctionIndex.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/supplementary/AnnotatedJunctionIndex.java new file mode 100644 index 00000000000..7e27be95626 --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/supplementary/AnnotatedJunctionIndex.java @@ -0,0 +1,55 @@ +package com.hartwig.hmftools.tars.liftback.supplementary; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import com.hartwig.hmftools.common.region.ChrBaseRegion; +import com.hartwig.hmftools.common.region.BasePosition; + +// Three views of an annotated-junction set: exact membership, by-intron-start, and by-intron-end. +// All three are built from the same Set in one pass. +public class AnnotatedJunctionIndex +{ + private final Set mJunctions; + private final Map> mByStart; + private final Map> mByEnd; + + public AnnotatedJunctionIndex(final Set junctions) + { + mJunctions = junctions != null ? junctions : new HashSet<>(); + mByStart = new HashMap<>(); + mByEnd = new HashMap<>(); + for(ChrBaseRegion intron : mJunctions) + { + mByStart.computeIfAbsent(new BasePosition(intron.Chromosome, intron.start()), + k -> new ArrayList<>()).add(intron); + mByEnd.computeIfAbsent(new BasePosition(intron.Chromosome, intron.end()), + k -> new ArrayList<>()).add(intron); + } + } + + public boolean contains(final ChrBaseRegion intron) + { + return mJunctions.contains(intron); + } + + public int size() + { + return mJunctions.size(); + } + + public List introByStart(final String chromosome, final int intronStart) + { + return mByStart.getOrDefault(new BasePosition(chromosome, intronStart), Collections.emptyList()); + } + + public List introByEnd(final String chromosome, final int intronEnd) + { + return mByEnd.getOrDefault(new BasePosition(chromosome, intronEnd), Collections.emptyList()); + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/supplementary/AnnotatedJunctionLoader.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/supplementary/AnnotatedJunctionLoader.java new file mode 100644 index 00000000000..f1c60a37215 --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/supplementary/AnnotatedJunctionLoader.java @@ -0,0 +1,110 @@ +package com.hartwig.hmftools.tars.liftback.supplementary; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import com.hartwig.hmftools.common.region.BaseRegion; +import com.hartwig.hmftools.common.region.ChrBaseRegion; +import com.hartwig.hmftools.common.ensemblcache.EnsemblDataCache; +import com.hartwig.hmftools.common.gene.ExonData; +import com.hartwig.hmftools.common.gene.GeneData; +import com.hartwig.hmftools.common.gene.TranscriptData; +import com.hartwig.hmftools.common.genome.refgenome.RefGenomeVersion; +import com.hartwig.hmftools.tars.common.ContigEntry; + +// Derives annotated splice junctions from the ensembl data cache into a Set for O(1) lookup. +// Pairs adjacent exons of each transcript to derive intron coords; duplicates across transcripts collapse in the Set. +public final class AnnotatedJunctionLoader +{ + private AnnotatedJunctionLoader() { } + + public static Set load(final String ensemblDir, final RefGenomeVersion refGenomeVersion) + { + EnsemblDataCache ensemblDataCache = new EnsemblDataCache(ensemblDir, refGenomeVersion); + ensemblDataCache.setRequiredData(true, false, false, false); + ensemblDataCache.load(false); + return deriveIntrons(ensemblDataCache, refGenomeVersion); + } + + // derive the same intron set from the contig sidecar: each entry's exonSpans are a multi-exon transcript's + // genomic exons. Single-exon transcripts contribute no junctions, so the sidecar misses nothing here. + public static Set fromContigEntries(final List entries) + { + Set introns = new HashSet<>(); + for(final ContigEntry entry : entries) + { + addSpanIntrons(introns, entry.chromosome(), entry.exonSpans()); + } + return introns; + } + + private static void addSpanIntrons(final Set introns, final String chromosome, final List exonSpans) + { + if(exonSpans.size() < 2) + { + return; + } + + List exons = new ArrayList<>(exonSpans); + exons.sort(Comparator.comparingInt(BaseRegion::start)); + + for(int i = 0; i < exons.size() - 1; ++i) + { + int intronStart = exons.get(i).end() + 1; + int intronEnd = exons.get(i + 1).start() - 1; + if(intronEnd < intronStart) + continue; // abutting exons -> no intron + + introns.add(new ChrBaseRegion(chromosome, intronStart, intronEnd)); + } + } + + public static Set deriveIntrons(final EnsemblDataCache ensemblDataCache, final RefGenomeVersion refGenomeVersion) + { + Set introns = new HashSet<>(); + for(final List genes : ensemblDataCache.getChrGeneDataMap().values()) + { + for(final GeneData gene : genes) + { + List transcripts = ensemblDataCache.getTranscripts(gene.GeneId); + if(transcripts == null) + continue; + + String chromosome = refGenomeVersion.versionedChromosome(gene.Chromosome); + for(final TranscriptData transcript : transcripts) + { + addTranscriptIntrons(introns, chromosome, transcript.exons()); + } + } + } + return introns; + } + + private static void addTranscriptIntrons(final Set introns, final String chromosome, + final List transcriptExons) + { + if(transcriptExons.size() < 2) + { + return; + } + + List exons = new ArrayList<>(transcriptExons); + exons.sort(Comparator.comparingInt(exon -> exon.Rank)); + + for(int i = 0; i < exons.size() - 1; ++i) + { + ExonData first = exons.get(i); + ExonData second = exons.get(i + 1); + // Rank follows transcription direction, not genomic coords -- derive intron bounds from coords directly. + int intronStart = Math.min(first.End, second.End) + 1; + int intronEnd = Math.max(first.Start, second.Start) - 1; + if(intronEnd < intronStart) + continue; // abutting or overlapping exons -> no intron + + introns.add(new ChrBaseRegion(chromosome, intronStart, intronEnd)); + } + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/supplementary/RefSequenceSource.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/supplementary/RefSequenceSource.java new file mode 100644 index 00000000000..d030abe9a8b --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/supplementary/RefSequenceSource.java @@ -0,0 +1,9 @@ +package com.hartwig.hmftools.tars.liftback.supplementary; + +// Reference-sequence interface for the supplementary-resolve ref-verify path; decouples production FASTA from tests. +// Coordinates are 1-based inclusive. Returns null for unknown chromosomes or out-of-range queries. +@FunctionalInterface +public interface RefSequenceSource +{ + byte[] getBases(String chromosome, int posStart, int posEnd); +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/supplementary/SpliceMotif.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/supplementary/SpliceMotif.java new file mode 100644 index 00000000000..db621dcffb2 --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/supplementary/SpliceMotif.java @@ -0,0 +1,44 @@ +package com.hartwig.hmftools.tars.liftback.supplementary; + +// Classifies donor/acceptor 2-base flanks: GT-AG canonical (~99%), GC-AG/AT-AC semi-canonical. +// Strand is unknown at scan time so reverse-complement motifs (CT-AC, CT-GC, GT-AT) are accepted. +public final class SpliceMotif +{ + private SpliceMotif() { } + + public static Tier classify(final byte[] donorBases, final byte[] acceptorBases) + { + if(donorBases == null || donorBases.length != 2) + { + return Tier.NONE; + } + if(acceptorBases == null || acceptorBases.length != 2) + { + return Tier.NONE; + } + + char d0 = upper(donorBases[0]); + char d1 = upper(donorBases[1]); + char a0 = upper(acceptorBases[0]); + char a1 = upper(acceptorBases[1]); + + if((d0 == 'G' && d1 == 'T' && a0 == 'A' && a1 == 'G') + || (d0 == 'C' && d1 == 'T' && a0 == 'A' && a1 == 'C')) + return Tier.CANONICAL; + + if((d0 == 'G' && d1 == 'C' && a0 == 'A' && a1 == 'G') + || (d0 == 'C' && d1 == 'T' && a0 == 'G' && a1 == 'C')) + return Tier.SEMI_CANONICAL; + + if((d0 == 'A' && d1 == 'T' && a0 == 'A' && a1 == 'C') + || (d0 == 'G' && d1 == 'T' && a0 == 'A' && a1 == 'T')) + return Tier.SEMI_CANONICAL; + + return Tier.NONE; + } + + private static char upper(final byte b) + { + return (char) (b & ~0x20); + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/supplementary/SupplementaryCandidate.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/supplementary/SupplementaryCandidate.java new file mode 100644 index 00000000000..99c1df4e08a --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/supplementary/SupplementaryCandidate.java @@ -0,0 +1,34 @@ +package com.hartwig.hmftools.tars.liftback.supplementary; + +import java.util.Collections; +import java.util.List; + +import com.hartwig.hmftools.common.region.ChrBaseRegion; + +// Input DTO for SupplementaryResolver, expressed as plain values so the resolver is testable without SAMRecord. +// readBases is only needed for the ref-verify path. mateHintIntrons, when set, biases junction snap toward the +// partner mate's already-resolved junctions. +// NOTE: the generated equals/hashCode compare readBases by array identity, not content; do not use this record +// as a Set/Map key or rely on value equality (it is only ever passed to resolve() as a transient input). +public record SupplementaryCandidate( + String chromosome, boolean forwardStrand, int readLength, int primaryStart, String primaryCigar, + int primaryMapq, List supplementaries, byte[] readBases, + List mateHintIntrons) +{ + public SupplementaryCandidate + { + if(mateHintIntrons == null) + { + mateHintIntrons = Collections.emptyList(); + } + } + + public SupplementaryCandidate( + final String chromosome, final boolean forwardStrand, final int readLength, + final int primaryStart, final String primaryCigar, final int primaryMapq, + final List supplementaries) + { + this(chromosome, forwardStrand, readLength, primaryStart, primaryCigar, primaryMapq, + supplementaries, null, Collections.emptyList()); + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/supplementary/SupplementaryConfig.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/supplementary/SupplementaryConfig.java new file mode 100644 index 00000000000..5efd0eafa85 --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/supplementary/SupplementaryConfig.java @@ -0,0 +1,55 @@ +package com.hartwig.hmftools.tars.liftback.supplementary; + +import static com.hartwig.hmftools.tars.common.TarsConstants.DEFAULT_MAX_BOUNDARY_SHIFT; +import static com.hartwig.hmftools.tars.common.TarsConstants.DEFAULT_MAX_SUPP_MERGES; +import static com.hartwig.hmftools.tars.common.TarsConstants.DEFAULT_MAX_INTRON_LENGTH; +import static com.hartwig.hmftools.tars.common.TarsConstants.DEFAULT_MIN_INTRON_LENGTH; +import static com.hartwig.hmftools.tars.common.TarsConstants.DEFAULT_SOFTCLIP_TOLERANCE; + +// Config for SupplementaryResolver. Defaults for the annotated-junction policy: +// min intron 21, max intron 1_000_000. +public class SupplementaryConfig +{ + public final boolean Enabled; + public final int MinIntronLength; + public final int MaxIntronLength; + public final int MaxSuppMerges; + public final boolean AnnotatedOnly; + public final int SoftclipTolerance; + public final int MaxBoundaryShift; + + public SupplementaryConfig( + final boolean enabled, final int minIntronLength, final int maxIntronLength, + final int maxSuppMerges, final boolean annotatedOnly, + final int softclipTolerance, final int maxBoundaryShift) + { + Enabled = enabled; + MinIntronLength = minIntronLength; + MaxIntronLength = maxIntronLength; + MaxSuppMerges = maxSuppMerges; + AnnotatedOnly = annotatedOnly; + SoftclipTolerance = softclipTolerance; + MaxBoundaryShift = maxBoundaryShift; + } + + public static SupplementaryConfig defaults() + { + // tolerance=0 preserves strict-complementary semantics for older tests; production uses enabledDefaults(). + return new SupplementaryConfig( + false, + DEFAULT_MIN_INTRON_LENGTH, DEFAULT_MAX_INTRON_LENGTH, + DEFAULT_MAX_SUPP_MERGES, + true, 0, 0); + } + + public static SupplementaryConfig enabledDefaults() + { + // AnnotatedOnly=false: merge builds a better alignment from existing BWA records, so missing + // annotation shouldn't block it. Annotated snap preferred when available; otherwise trust-primary. + return new SupplementaryConfig( + true, + DEFAULT_MIN_INTRON_LENGTH, DEFAULT_MAX_INTRON_LENGTH, + DEFAULT_MAX_SUPP_MERGES, + false, DEFAULT_SOFTCLIP_TOLERANCE, DEFAULT_MAX_BOUNDARY_SHIFT); + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/supplementary/SupplementaryRecord.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/supplementary/SupplementaryRecord.java new file mode 100644 index 00000000000..45ef53e2d2a --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/supplementary/SupplementaryRecord.java @@ -0,0 +1,8 @@ +package com.hartwig.hmftools.tars.liftback.supplementary; + +// Supplementary record under consideration as a merge partner. index is its position in the caller's +// supplementary list so the resolver can report which supps to drop without exposing SAMRecord. +public record SupplementaryRecord( + int index, String chromosome, boolean forwardStrand, int start, String cigar, int mapq) +{ +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/supplementary/SupplementaryRejectReason.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/supplementary/SupplementaryRejectReason.java new file mode 100644 index 00000000000..d3e6ec1e790 --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/supplementary/SupplementaryRejectReason.java @@ -0,0 +1,25 @@ +package com.hartwig.hmftools.tars.liftback.supplementary; + +// Drives per-decision counters in SupplementaryStatistics. +public enum SupplementaryRejectReason +{ + NO_TERMINAL_SOFTCLIP, // primary cigar has no leading or trailing S to extend across + NO_MATCHING_SUPP, // no supplementary had a complementary cigar shape + DIFFERENT_CHROMOSOME, + OPPOSITE_STRAND, + READ_COVERAGE_OVERLAP, // primary's M + supp's M overlap on the read + READ_COVERAGE_GAP, // primary's M + supp's M leave a gap on the read + INTRON_TOO_SHORT, + INTRON_TOO_LONG, + SHORT_ANCHOR, // primary or supp matched portion < MinAnchorOverhang + NOVEL_JUNCTION, // candidate intron not in annotated set (and AnnotatedOnly is true) + COMPLEX_CIGAR_SHAPE, // hard clip, indel adjacent to softclip boundary, etc. + AMBIGUOUS_SUPP_CHOICE, // multiple equally-good candidate supps; refuse to guess + MULTIPLE_SUPPS_IN_REACH, // more than one supp within merge reach; refuse to guess which splice + // ref-verify path: primary has terminal softclip but no matching supp + REF_VERIFY_NO_CANDIDATE_EXON, // no annotated junction's adjacent exon lines up with the softclip + REF_VERIFY_MISMATCH_TOO_HIGH, // softclipped read bases don't match the candidate exon's ref bases + REF_VERIFY_NO_REF_SOURCE, // ref-verify requested but no RefSequenceSource configured + REF_VERIFY_AMBIGUOUS, // multiple annotated downstream/upstream exons match; refuse to guess + REF_VERIFY_SHORT_PARTIAL_RUN // partial match (outer residual left clipped) shorter than MinPartialMatchRun +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/supplementary/SupplementaryResolver.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/supplementary/SupplementaryResolver.java new file mode 100644 index 00000000000..3949d80be61 --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/supplementary/SupplementaryResolver.java @@ -0,0 +1,1116 @@ +package com.hartwig.hmftools.tars.liftback.supplementary; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import com.hartwig.hmftools.common.bam.CigarUtils; +import com.hartwig.hmftools.common.region.ChrBaseRegion; +import com.hartwig.hmftools.tars.liftback.overhang.BoundaryReclaim; + +import htsjdk.samtools.CigarElement; +import htsjdk.samtools.CigarOperator; + +// Merges a primary's terminal softclip with an annotated-intron-spanning supplementary into a +// single spliced primary (N op). Chains up to SupplementaryConfig.MaxSuppMerges merges per read. +public class SupplementaryResolver +{ + private final AnnotatedJunctionIndex mAnnotatedIndex; + private final RefSequenceSource mRefSource; + private final SupplementaryConfig mConfig; + private final SupplementaryStatistics mStatistics; + + public SupplementaryResolver(final Set annotatedJunctions, final SupplementaryConfig config) + { + this(new AnnotatedJunctionIndex(annotatedJunctions != null ? annotatedJunctions : new HashSet<>()), + null, config); + } + + public SupplementaryResolver( + final AnnotatedJunctionIndex annotatedIndex, final RefSequenceSource refSource, + final SupplementaryConfig config) + { + mAnnotatedIndex = annotatedIndex != null ? annotatedIndex : new AnnotatedJunctionIndex(new HashSet<>()); + mRefSource = refSource; + mConfig = config; + mStatistics = new SupplementaryStatistics(config.MaxSuppMerges); + } + + public SupplementaryStatistics statistics() + { + return mStatistics; + } + + public SupplementaryResult resolve(final SupplementaryCandidate candidate) + { + if(!mConfig.Enabled) + { + return SupplementaryResult.noMerge(SupplementaryRejectReason.NO_MATCHING_SUPP); + } + + if(candidate.supplementaries().isEmpty()) + { + return tryRefVerifyOnly(candidate); + } + + mStatistics.countCandidate(); + + List primaryCigar = CigarUtils.cigarElementsFromStr(candidate.primaryCigar()); + if(CigarUtils.hasHardClip(primaryCigar)) + { + mStatistics.countReject(SupplementaryRejectReason.COMPLEX_CIGAR_SHAPE); + return SupplementaryResult.noMerge(SupplementaryRejectReason.COMPLEX_CIGAR_SHAPE); + } + + int primaryStart = candidate.primaryStart(); + List remaining = new ArrayList<>(candidate.supplementaries()); + List dropped = new ArrayList<>(); + List introns = new ArrayList<>(); + SupplementaryRejectReason lastReject = null; + int chainDepth = 0; + + while(chainDepth < mConfig.MaxSuppMerges && !remaining.isEmpty()) + { + boolean primaryHasLeadingS = !primaryCigar.isEmpty() + && primaryCigar.get(0).getOperator() == CigarOperator.S; + boolean primaryHasTrailingS = !primaryCigar.isEmpty() + && primaryCigar.get(primaryCigar.size() - 1).getOperator() == CigarOperator.S; + if(!primaryHasLeadingS && !primaryHasTrailingS) + { + if(chainDepth == 0) + { + return SupplementaryResult.noMerge(SupplementaryRejectReason.NO_TERMINAL_SOFTCLIP); + } + break; + } + + MergeOutcome attempt = pickBestSupplementary( + candidate, primaryStart, primaryCigar, remaining); + + if(!attempt.isSuccess()) + { + if(chainDepth == 0) + { + return SupplementaryResult.noMerge(attempt.Reject); + } + lastReject = attempt.Reject; + break; + } + + primaryStart = attempt.MergedStart; + primaryCigar = attempt.MergedCigar; + dropped.add(attempt.MergedSupp.index()); + introns.add(attempt.IntroducedIntron); + remaining.remove(attempt.MergedSupp); + ++chainDepth; + } + + if(chainDepth == 0) + { + // Belt and braces: initial-iteration failures should have returned above. + return SupplementaryResult.noMerge(lastReject != null ? lastReject : SupplementaryRejectReason.NO_MATCHING_SUPP); + } + + mStatistics.countMergedChain(chainDepth); + return new SupplementaryResult( + true, CigarUtils.cigarElementsToStr(primaryCigar), primaryStart, + dropped, introns, chainDepth, null); + } + + // No supplementary: look up annotated junctions abutting the softclip boundary and accept if + // softclipped bases match the candidate exon reference within tolerance. + private SupplementaryResult tryRefVerifyOnly(final SupplementaryCandidate candidate) + { + if(mRefSource == null || candidate.readBases() == null) + { + return SupplementaryResult.noMerge(SupplementaryRejectReason.NO_MATCHING_SUPP); + } + + mStatistics.countCandidate(); + + List primaryCigar = CigarUtils.cigarElementsFromStr(candidate.primaryCigar()); + if(CigarUtils.hasHardClip(primaryCigar)) + { + mStatistics.countReject(SupplementaryRejectReason.COMPLEX_CIGAR_SHAPE); + return SupplementaryResult.noMerge(SupplementaryRejectReason.COMPLEX_CIGAR_SHAPE); + } + if(primaryCigar.isEmpty()) + { + return SupplementaryResult.noMerge(SupplementaryRejectReason.NO_TERMINAL_SOFTCLIP); + } + + boolean trailingS = primaryCigar.get(primaryCigar.size() - 1).getOperator() == CigarOperator.S; + boolean leadingS = primaryCigar.get(0).getOperator() == CigarOperator.S; + if(!trailingS && !leadingS) + { + return SupplementaryResult.noMerge(SupplementaryRejectReason.NO_TERMINAL_SOFTCLIP); + } + + // Both-end clips: try longer clip first (more likely the splice tail). First win is taken. + boolean[] sides; + if(trailingS && leadingS) + { + boolean trailingFirst = + CigarUtils.rightSoftClipLength(primaryCigar) >= CigarUtils.leftSoftClipLength(primaryCigar); + sides = new boolean[] { trailingFirst, !trailingFirst }; + } + else + { + sides = new boolean[] { trailingS }; + } + + boolean anyCandidate = false; + SupplementaryResult lastFailure = SupplementaryResult.noMerge(SupplementaryRejectReason.REF_VERIFY_MISMATCH_TOO_HIGH); + for(boolean rightExtend : sides) + { + SupplementaryResult sideResult = attemptRefVerifySide(candidate, primaryCigar, rightExtend); + if(sideResult.merged()) + { + return sideResult; + } + if(sideResult.rejectReason() != SupplementaryRejectReason.REF_VERIFY_NO_CANDIDATE_EXON) + { + anyCandidate = true; + lastFailure = sideResult; + } + } + + if(!anyCandidate) + { + mStatistics.countReject(SupplementaryRejectReason.REF_VERIFY_NO_CANDIDATE_EXON); + return SupplementaryResult.noMerge(SupplementaryRejectReason.REF_VERIFY_NO_CANDIDATE_EXON); + } + return lastFailure; + } + + // Ref-verify one terminal softclip against annotated donors/acceptors with boundary snap. + // Does NOT count the no-candidate reject; tryRefVerifyOnly aggregates across both ends. + private SupplementaryResult attemptRefVerifySide( + final SupplementaryCandidate candidate, final List primaryCigar, final boolean rightExtend) + { + if(opAdjacentToSoftClip(primaryCigar, !rightExtend)) + { + return SupplementaryResult.noMerge(SupplementaryRejectReason.COMPLEX_CIGAR_SHAPE); + } + + int softclipLen = rightExtend + ? CigarUtils.rightSoftClipLength(primaryCigar) + : CigarUtils.leftSoftClipLength(primaryCigar); + int primaryAnchor = rightExtend + ? trailingMatchedRun(primaryCigar) + : leadingMatchedRun(primaryCigar); + + // BWA often over-extends a few bases past the true exon boundary. Snap back up to + // MaxBoundaryShift (smallest shift first), rolling over-extension into the softclip. + int primaryRefEnd = candidate.primaryStart() + CigarUtils.cigarAlignedLength(primaryCigar) - 1; + boolean anyCandidate = false; + SupplementaryResult lastFailure = SupplementaryResult.noMerge(SupplementaryRejectReason.REF_VERIFY_MISMATCH_TOO_HIGH); + + for(int shift = 0; shift <= mConfig.MaxBoundaryShift; ++shift) + { + if(primaryAnchor - shift < 1) + break; + List shiftedCigar = shift == 0 + ? primaryCigar + : shiftBoundaryIntoSoftclip(primaryCigar, shift, rightExtend); + if(shiftedCigar == null) + break; + + int boundary = rightExtend ? (primaryRefEnd + 1 - shift) : (candidate.primaryStart() - 1 + shift); + List candidates = rightExtend + ? mAnnotatedIndex.introByStart(candidate.chromosome(), boundary) + : mAnnotatedIndex.introByEnd(candidate.chromosome(), boundary); + if(candidates.isEmpty()) + continue; + + anyCandidate = true; + SupplementaryResult result = verifyAgainstCandidates( + candidate, shiftedCigar, candidates, softclipLen + shift, rightExtend); + if(result.merged()) + { + return result; + } + lastFailure = result; + } + + if(!anyCandidate) + { + return SupplementaryResult.noMerge(SupplementaryRejectReason.REF_VERIFY_NO_CANDIDATE_EXON); + } + return lastFailure; + } + + // Moves 'shift' bases from the exon-edge M into the adjacent softclip to snap back the boundary. + // Returns null when the edge op isn't M or trimming would leave nothing matched. + private static List shiftBoundaryIntoSoftclip( + final List cigar, final int shift, final boolean rightExtend) + { + List shifted = new ArrayList<>(cigar); + int last = shifted.size() - 1; + int softIdx = rightExtend ? last : 0; + int matchIdx = rightExtend ? last - 1 : 1; + if(matchIdx < 0 || matchIdx >= shifted.size()) + { + return null; + } + + CigarElement softElement = shifted.get(softIdx); + CigarElement matchElement = shifted.get(matchIdx); + if(matchElement.getOperator() != CigarOperator.M && matchElement.getOperator() != CigarOperator.EQ) + { + return null; + } + if(matchElement.getLength() - shift < 1) + { + return null; + } + + shifted.set(matchIdx, new CigarElement(matchElement.getLength() - shift, matchElement.getOperator())); + shifted.set(softIdx, new CigarElement(softElement.getLength() + shift, CigarOperator.S)); + return shifted; + } + + private SupplementaryResult verifyAgainstCandidates( + final SupplementaryCandidate candidate, final List primaryCigar, + final List candidates, final int softclipLen, final boolean rightExtend) + { + if(candidates.isEmpty()) + { + mStatistics.countReject(SupplementaryRejectReason.REF_VERIFY_NO_CANDIDATE_EXON); + return SupplementaryResult.noMerge(SupplementaryRejectReason.REF_VERIFY_NO_CANDIDATE_EXON); + } + + byte[] readBases = candidate.readBases(); + byte[] softclipBases = new byte[softclipLen]; + if(rightExtend) + { + System.arraycopy(readBases, readBases.length - softclipLen, softclipBases, 0, softclipLen); + } + else + { + System.arraycopy(readBases, 0, softclipBases, 0, softclipLen); + } + + // Match from the junction-proximal end (bwa-mem score walk). The outer softclip residual stays + // soft-clipped since bwa often carries adapter/low-quality bases there (e.g. 19S132M -> + // 6S15M..N..130M, not 21M..N..130M). Proximal end is the high index for a leading softclip. + boolean proximalAtEnd = !rightExtend; + ChrBaseRegion chosen = null; + int chosenRun = 0; + int chosenMismatches = Integer.MAX_VALUE; + boolean ambiguous = false; + + for(ChrBaseRegion candidateIntron : candidates) + { + int intronLength = candidateIntron.end() - candidateIntron.start() + 1; + if(intronLength < mConfig.MinIntronLength || intronLength > mConfig.MaxIntronLength) + continue; + + int refStart; + int refEnd; + if(rightExtend) + { + refStart = candidateIntron.end() + 1; + refEnd = candidateIntron.end() + softclipLen; + } + else + { + refStart = candidateIntron.start() - softclipLen; + refEnd = candidateIntron.start() - 1; + } + + byte[] refBases = mRefSource.getBases(candidate.chromosome(), refStart, refEnd); + if(refBases == null || refBases.length != softclipLen) + continue; + + int run = proximalScoringRun(softclipBases, refBases, rightExtend); + if(run == 0) + continue; + int mismatches = mismatchesInRun(softclipBases, refBases, softclipLen, run, proximalAtEnd); + + // Longest run wins; tie-break fewest mismatches. Two introns tying both -> ambiguous -> reject. + if(run > chosenRun || (run == chosenRun && mismatches < chosenMismatches)) + { + chosen = candidateIntron; + chosenRun = run; + chosenMismatches = mismatches; + ambiguous = false; + } + else if(run == chosenRun && mismatches == chosenMismatches && !candidateIntron.equals(chosen)) + { + ambiguous = true; + } + } + + if(chosen == null) + { + mStatistics.countReject(SupplementaryRejectReason.REF_VERIFY_MISMATCH_TOO_HIGH); + return SupplementaryResult.noMerge(SupplementaryRejectReason.REF_VERIFY_MISMATCH_TOO_HIGH); + } + if(ambiguous) + { + mStatistics.countReject(SupplementaryRejectReason.REF_VERIFY_AMBIGUOUS); + return SupplementaryResult.noMerge(SupplementaryRejectReason.REF_VERIFY_AMBIGUOUS); + } + + return buildRefVerifyMerge(candidate, primaryCigar, chosen, chosenRun, softclipLen, rightExtend); + } + + private SupplementaryResult buildRefVerifyMerge( + final SupplementaryCandidate candidate, final List primaryCigar, + final ChrBaseRegion chosen, final int matchedRun, final int softclipLen, final boolean rightExtend) + { + int intronLength = chosen.end() - chosen.start() + 1; + int residualSoftclip = softclipLen - matchedRun; + List merged = new ArrayList<>(primaryCigar.size() + 3); + if(rightExtend) + { + for(int i = 0; i < primaryCigar.size() - 1; ++i) + { + merged.add(primaryCigar.get(i)); + } + merged.add(new CigarElement(intronLength, CigarOperator.N)); + merged.add(new CigarElement(matchedRun, CigarOperator.M)); + if(residualSoftclip > 0) + { + merged.add(new CigarElement(residualSoftclip, CigarOperator.S)); + } + } + else + { + if(residualSoftclip > 0) + { + merged.add(new CigarElement(residualSoftclip, CigarOperator.S)); + } + merged.add(new CigarElement(matchedRun, CigarOperator.M)); + merged.add(new CigarElement(intronLength, CigarOperator.N)); + for(int i = 1; i < primaryCigar.size(); ++i) + { + merged.add(primaryCigar.get(i)); + } + } + + int mergedStart = rightExtend ? candidate.primaryStart() : (chosen.start() - matchedRun); + mStatistics.countMergedChain(1); + return new SupplementaryResult( + true, CigarUtils.cigarElementsToStr(merged), mergedStart, + Collections.emptyList(), Collections.singletonList(chosen), + 1, null); + } + + // Longest run matching ref from the junction-proximal end, scored with the shared bwa-mem model + // (BoundaryReclaim: match +1, mismatch -4) so ref-verify uses the same matching as collapse/tail-extend. + // Trailing clips walk forward from index 0 (junction-proximal); leading clips reverse both arrays so the + // walk runs from the high-index proximal end outward. + private static int proximalScoringRun(final byte[] softclipBases, final byte[] refBases, final boolean rightExtend) + { + if(rightExtend) + { + return BoundaryReclaim.maxScoringPrefix(softclipBases, refBases); + } + return BoundaryReclaim.maxScoringPrefix(BoundaryReclaim.reversed(softclipBases), BoundaryReclaim.reversed(refBases)); + } + + private static int mismatchesInRun( + final byte[] softclipBases, final byte[] refBases, final int len, final int run, final boolean proximalAtEnd) + { + int mismatches = 0; + for(int k = 1; k <= run; ++k) + { + int idx = proximalAtEnd ? (len - k) : (k - 1); + if(!basesEqualIgnoreCase(softclipBases[idx], refBases[idx])) + { + ++mismatches; + } + } + return mismatches; + } + + private Tier classifyJunctionTier(final ChrBaseRegion candidateIntron) + { + if(mAnnotatedIndex.contains(candidateIntron)) + { + return Tier.ANNOTATED; + } + if(mRefSource == null) + { + return Tier.NONE; + } + byte[] donor = mRefSource.getBases( + candidateIntron.Chromosome, candidateIntron.start(), candidateIntron.start() + 1); + byte[] acceptor = mRefSource.getBases( + candidateIntron.Chromosome, candidateIntron.end() - 1, candidateIntron.end()); + return SpliceMotif.classify(donor, acceptor); + } + + private static boolean basesEqualIgnoreCase(final byte a, final byte b) + { + if(a == b) + { + return true; + } + return (a & ~0x20) == (b & ~0x20); // mask ASCII case bit; valid for letters, cheap + } + + private MergeOutcome pickBestSupplementary( + final SupplementaryCandidate candidate, final int primaryStart, + final List primaryCigar, final List supps) + { + MergeOutcome best = null; + SupplementaryRejectReason lastReject = null; + // Ambiguous only when one terminal side draws more than one supp. + int rightReach = 0; + int leftReach = 0; + + for(SupplementaryRecord supp : supps) + { + MergeOutcome attempt = tryMerge(candidate, primaryStart, primaryCigar, supp); + if(!attempt.isSuccess()) + { + mStatistics.countReject(attempt.Reject); + lastReject = attempt.Reject; + continue; + } + + if(attempt.RightExtend) + { + ++rightReach; + } + else + { + ++leftReach; + } + + if(best == null) + { + best = attempt; + } + else if(isBetterCandidate(attempt, best)) + { + best = attempt; + } + } + + if(best == null) + { + return MergeOutcome.reject(lastReject != null ? lastReject : SupplementaryRejectReason.NO_MATCHING_SUPP); + } + + if(rightReach > 1 || leftReach > 1) + { + mStatistics.countReject(SupplementaryRejectReason.MULTIPLE_SUPPS_IN_REACH); + return MergeOutcome.reject(SupplementaryRejectReason.MULTIPLE_SUPPS_IN_REACH); + } + + return best; + } + + // Higher MAPQ wins, then smaller intron. >1 within reach is rejected by the caller. + private static boolean isBetterCandidate(final MergeOutcome a, final MergeOutcome b) + { + if(a.MergedSupp.mapq() != b.MergedSupp.mapq()) + { + return a.MergedSupp.mapq() > b.MergedSupp.mapq(); + } + return intronLength(a.IntroducedIntron) < intronLength(b.IntroducedIntron); + } + + private static int intronLength(final ChrBaseRegion intron) + { + return intron.end() - intron.start() + 1; + } + + private MergeOutcome tryMerge( + final SupplementaryCandidate candidate, final int primaryStart, + final List primaryCigar, final SupplementaryRecord supp) + { + if(!candidate.chromosome().equals(supp.chromosome())) + { + return MergeOutcome.reject(SupplementaryRejectReason.DIFFERENT_CHROMOSOME); + } + + if(candidate.forwardStrand() != supp.forwardStrand()) + { + return MergeOutcome.reject(SupplementaryRejectReason.OPPOSITE_STRAND); + } + + List suppCigar = CigarUtils.cigarElementsFromStr(supp.cigar()); + if(CigarUtils.hasHardClip(suppCigar)) + { + return MergeOutcome.reject(SupplementaryRejectReason.COMPLEX_CIGAR_SHAPE); + } + + // ContigTranslator can expand a cross-exon M into M-N-M. If the post-N M coincidentally + // overlaps the primary's span, clamp the supp to its primary-distal anchor before merge logic. + int primaryRefEnd = primaryStart + CigarUtils.cigarAlignedLength(primaryCigar) - 1; + int suppStart = supp.start(); + ClampedSupp clamped = clampSuppToPrimaryBoundary(suppCigar, suppStart, primaryStart, primaryRefEnd); + if(clamped != null) + { + suppCigar = clamped.Cigar; + suppStart = clamped.Start; + mStatistics.countSuppClamp(); + } + + int primaryLeadS = CigarUtils.leftSoftClipLength(primaryCigar); + int primaryTrailS = CigarUtils.rightSoftClipLength(primaryCigar); + int suppLeadS = CigarUtils.leftSoftClipLength(suppCigar); + int suppTrailS = CigarUtils.rightSoftClipLength(suppCigar); + + boolean rightExtend = primaryTrailS > 0 && suppLeadS > 0; + boolean leftExtend = primaryLeadS > 0 && suppTrailS > 0; + + if(!rightExtend && !leftExtend) + { + return MergeOutcome.reject(SupplementaryRejectReason.NO_MATCHING_SUPP); + } + + if(rightExtend && leftExtend) + { + int suppRefEnd = suppStart + CigarUtils.cigarAlignedLength(suppCigar) - 1; + if(suppStart > primaryRefEnd && suppRefEnd >= primaryStart) + { + leftExtend = false; + } + else if(suppRefEnd < primaryStart && suppStart <= primaryRefEnd) + { + rightExtend = false; + } + else + { + return MergeOutcome.reject(SupplementaryRejectReason.COMPLEX_CIGAR_SHAPE); + } + } + + Side primarySide = Side.of(primaryStart, primaryCigar); + Side suppSide = Side.of(suppStart, suppCigar); + boolean primaryIsUpstream = rightExtend; + Side up = primaryIsUpstream ? primarySide : suppSide; + Side down = primaryIsUpstream ? suppSide : primarySide; + + return mergeJunction(candidate, up, down, primaryIsUpstream, supp); + } + + // Direction-agnostic merge: validates anchor pair, scores snap points by junction tier, falls + // back to mate-hint then trust-primary. supp is threaded as result payload only. + private MergeOutcome mergeJunction( + final SupplementaryCandidate candidate, final Side up, final Side down, + final boolean primaryIsUpstream, final SupplementaryRecord supp) + { + if(CigarUtils.cigarBaseLength(up.Cigar) != candidate.readLength() + || CigarUtils.cigarBaseLength(down.Cigar) != candidate.readLength()) + return MergeOutcome.reject(SupplementaryRejectReason.COMPLEX_CIGAR_SHAPE); + + if(opAdjacentToSoftClip(up.Cigar, false) || opAdjacentToSoftClip(down.Cigar, true)) + { + return MergeOutcome.reject(SupplementaryRejectReason.COMPLEX_CIGAR_SHAPE); + } + + int upMatchedRead = candidate.readLength() - up.TrailingS; + int overlap = upMatchedRead - down.LeadingS; + + if(overlap < 0) + { + return MergeOutcome.reject(SupplementaryRejectReason.READ_COVERAGE_GAP); + } + if(overlap > mConfig.SoftclipTolerance) + { + return MergeOutcome.reject(SupplementaryRejectReason.READ_COVERAGE_OVERLAP); + } + if(down.Start <= up.RefEnd) + { + return MergeOutcome.reject(SupplementaryRejectReason.READ_COVERAGE_OVERLAP); + } + + // Intron length is invariant under snap point L - check once up front. + int intronLength = (down.Start - 1 - up.RefEnd) + overlap; + if(intronLength < mConfig.MinIntronLength) + { + return MergeOutcome.reject(SupplementaryRejectReason.INTRON_TOO_SHORT); + } + if(intronLength > mConfig.MaxIntronLength) + { + return MergeOutcome.reject(SupplementaryRejectReason.INTRON_TOO_LONG); + } + + // Snap priority: an annotated boundary from the sidecar, else a canonical (then semi-canonical) splice + // motif; ties within the chosen tier are broken pseudo-randomly but deterministically (seeded by the read), + // so an ambiguous multi-option junction is distributed yet reproducible. The mate's resolved intron is a + // strong known-position hint, tried next. With no motif and no hint, the junction is placed at the midpoint + // of the ambiguous range (rounded down); if even that has no valid anchor split, the merge is not applied and + // the read keeps whatever placement was already chosen. + SnapPick pick = scanSnapPoints(candidate, up, down, upMatchedRead, primaryIsUpstream); + int chosenL = pick.L; + ChrBaseRegion chosenIntron = pick.Intron; + + if(chosenL == -1) + { + SnapPick hinted = scanMateHint(candidate, up, down, upMatchedRead, primaryIsUpstream); + chosenL = hinted.L; + chosenIntron = hinted.Intron; + } + if(chosenL == -1) + { + if(mConfig.AnnotatedOnly) + { + return MergeOutcome.reject(SupplementaryRejectReason.NOVEL_JUNCTION); + } + // midpoint of the ambiguous read range [down.LeadingS, upMatchedRead], rounded down. + chosenL = (upMatchedRead + down.LeadingS) / 2; + int upLossFb = upMatchedRead - chosenL; + int downLossFb = chosenL - down.LeadingS; + if(up.TrailingM < upLossFb || down.LeadingM < downLossFb) + return MergeOutcome.reject(SupplementaryRejectReason.SHORT_ANCHOR); + chosenIntron = new ChrBaseRegion( + candidate.chromosome(), up.RefEnd - upLossFb + 1, down.Start + downLossFb - 1); + } + + int upLoss = upMatchedRead - chosenL; + int downLoss = chosenL - down.LeadingS; + List merged = buildMergedCigar(up.Cigar, down.Cigar, upLoss, downLoss, intronLength); + return MergeOutcome.success(up.Start, merged, chosenIntron, supp, primaryIsUpstream); + } + + // Scores every valid snap point by junction tier and returns one at the highest tier present. When several + // snap points tie at that tier, the choice is pseudo-random but deterministic (seeded by the read), so an + // ambiguous junction with multiple equally-good positions is distributed across them yet reproducible run to + // run. Returns a miss when no snap point reaches a splice motif or annotated boundary. + private SnapPick scanSnapPoints( + final SupplementaryCandidate candidate, final Side up, final Side down, + final int upMatchedRead, final boolean primaryIsUpstream) + { + Tier bestTier = Tier.NONE; + List bestLs = new ArrayList<>(); + List bestIntrons = new ArrayList<>(); + + int startL = primaryIsUpstream ? upMatchedRead : down.LeadingS; + int endL = primaryIsUpstream ? down.LeadingS : upMatchedRead; + int step = primaryIsUpstream ? -1 : 1; + for(int L = startL; primaryIsUpstream ? L >= endL : L <= endL; L += step) + { + int upLoss = upMatchedRead - L; + int downLoss = L - down.LeadingS; + if(up.TrailingM < upLoss || down.LeadingM < downLoss) + continue; + + ChrBaseRegion candidateIntron = new ChrBaseRegion( + candidate.chromosome(), up.RefEnd - upLoss + 1, down.Start + downLoss - 1); + Tier tier = classifyJunctionTier(candidateIntron); + if(tier == Tier.NONE) + continue; + + int cmp = tier.compareTo(bestTier); + if(cmp > 0) + { + bestTier = tier; + bestLs.clear(); + bestIntrons.clear(); + } + if(cmp >= 0) + { + bestLs.add(L); + bestIntrons.add(candidateIntron); + } + } + + if(bestLs.isEmpty()) + { + return SnapPick.miss(); + } + int index = bestLs.size() == 1 ? 0 : Math.floorMod(snapSeed(candidate, up), bestLs.size()); + return new SnapPick(bestLs.get(index), bestIntrons.get(index)); + } + + // Deterministic per-read seed so an ambiguous multi-option snap is reproducible run to run: hashes the read + // bases (when available) with the upstream boundary so two reads over the same junction can pick differently. + private static int snapSeed(final SupplementaryCandidate candidate, final Side up) + { + int base = candidate.readBases() != null ? Arrays.hashCode(candidate.readBases()) : 0; + return 31 * base + up.RefEnd; + } + + // Use the mate's previously-resolved intron as a hint. Primary-upstream pins intron start; + // primary-downstream pins intron end. + private SnapPick scanMateHint( + final SupplementaryCandidate candidate, final Side up, final Side down, + final int upMatchedRead, final boolean primaryIsUpstream) + { + if(candidate.mateHintIntrons().isEmpty()) + { + return SnapPick.miss(); + } + + for(ChrBaseRegion hint : candidate.mateHintIntrons()) + { + if(!hint.Chromosome.equals(candidate.chromosome())) + continue; + int upLoss; + if(primaryIsUpstream) + { + upLoss = up.RefEnd - hint.start() + 1; + } + else + { + upLoss = upMatchedRead - (down.LeadingS + (hint.end() - (down.Start - 1))); + } + int L = upMatchedRead - upLoss; + if(L < down.LeadingS || L > upMatchedRead) + continue; + int downLoss = L - down.LeadingS; + if(up.TrailingM < upLoss || down.LeadingM < downLoss) + continue; + + int hintedIntronStart = primaryIsUpstream ? hint.start() : (up.RefEnd - upLoss + 1); + int hintedIntronEnd = primaryIsUpstream ? (down.Start + downLoss - 1) : hint.end(); + return new SnapPick(L, new ChrBaseRegion(candidate.chromosome(), hintedIntronStart, hintedIntronEnd)); + } + return SnapPick.miss(); + } + + private static List buildMergedCigar( + final List upCigar, final List downCigar, + final int upLoss, final int downLoss, final int intronLength) + { + List merged = new ArrayList<>(upCigar.size() + downCigar.size()); + for(int i = 0; i < upCigar.size() - 1; ++i) // upstream ops, excluding trailing S + { + if(i == upCigar.size() - 2 && upLoss > 0) + { + merged.add(new CigarElement(upCigar.get(i).getLength() - upLoss, upCigar.get(i).getOperator())); + } + else + { + merged.add(upCigar.get(i)); + } + } + merged.add(new CigarElement(intronLength, CigarOperator.N)); + for(int i = 1; i < downCigar.size(); ++i) // downstream ops, excluding leading S + { + if(i == 1 && downLoss > 0) + { + merged.add(new CigarElement(downCigar.get(i).getLength() - downLoss, downCigar.get(i).getOperator())); + } + else + { + merged.add(downCigar.get(i)); + } + } + return merged; + } + + // Length of the M/=/X run adjacent to the trailing softclip (e.g. "5M 50S" -> 5). + private static int trailingMatchedRun(final List elements) + { + for(int i = elements.size() - 1; i >= 0; --i) + { + CigarOperator op = elements.get(i).getOperator(); + if(op == CigarOperator.S) + continue; + if(op == CigarOperator.M || op == CigarOperator.EQ || op == CigarOperator.X) + { + return elements.get(i).getLength(); + } + return 0; + } + return 0; + } + + private static int leadingMatchedRun(final List elements) + { + for(int i = 0; i < elements.size(); ++i) + { + CigarOperator op = elements.get(i).getOperator(); + if(op == CigarOperator.S) + continue; + if(op == CigarOperator.M || op == CigarOperator.EQ || op == CigarOperator.X) + { + return elements.get(i).getLength(); + } + return 0; + } + return 0; + } + + private static boolean opAdjacentToSoftClip(final List elements, final boolean leadingSide) + { + if(elements.size() < 2) + { + return false; + } + if(leadingSide) + { + if(elements.get(0).getOperator() != CigarOperator.S) + { + return false; + } + CigarOperator next = elements.get(1).getOperator(); + return next == CigarOperator.I || next == CigarOperator.D; + } + else + { + int last = elements.size() - 1; + if(elements.get(last).getOperator() != CigarOperator.S) + { + return false; + } + CigarOperator prev = elements.get(last - 1).getOperator(); + return prev == CigarOperator.I || prev == CigarOperator.D; + } + } + + // Replaces the supp's post-N tail (or pre-N head) with a softclip when the M-after-N overlaps + // the primary's span (ContigTranslator M-N-M artifact double-counting read bases). Returns null + // when no internal N or no overlap. + private static ClampedSupp clampSuppToPrimaryBoundary( + final List suppCigar, final int suppStart, + final int primaryStart, final int primaryRefEnd) + { + boolean hasInternalN = false; + for(CigarElement e : suppCigar) + { + if(e.getOperator() == CigarOperator.N) + { + hasInternalN = true; + break; + } + } + if(!hasInternalN) + { + return null; + } + + int suppRefEnd = suppStart + CigarUtils.cigarAlignedLength(suppCigar) - 1; + if(suppStart < primaryStart) + { + return clampUpstreamSupp(suppCigar, suppStart, primaryStart); + } + if(suppRefEnd > primaryRefEnd) + { + return clampDownstreamSupp(suppCigar, suppStart, primaryRefEnd); + } + return null; + } + + private static ClampedSupp clampUpstreamSupp( + final List suppCigar, final int suppStart, final int primaryStart) + { + int refCursor = suppStart; + int readCursor = 0; + int firstNIdx = -1; + int readAtFirstN = 0; + + for(int i = 0; i < suppCigar.size(); i++) + { + CigarElement e = suppCigar.get(i); + CigarOperator op = e.getOperator(); + int len = e.getLength(); + + if(op == CigarOperator.N && firstNIdx == -1) + { + firstNIdx = i; + readAtFirstN = readCursor; + } + + if(firstNIdx != -1 && refCursor >= primaryStart + && (op == CigarOperator.M || op == CigarOperator.EQ || op == CigarOperator.X)) + { + return cutTailAt(suppCigar, suppStart, firstNIdx, readAtFirstN); + } + + if(op.consumesReferenceBases()) + { + refCursor += len; + } + if(op.consumesReadBases()) + { + readCursor += len; + } + } + return null; + } + + private static ClampedSupp clampDownstreamSupp( + final List suppCigar, final int suppStart, final int primaryRefEnd) + { + int refCursor = suppStart; + int readCursor = 0; + int lastBadNIdx = -1; + int readAfterLastBadN = 0; + int refAfterLastBadN = -1; + boolean prevMInsidePrimary = false; + + for(int i = 0; i < suppCigar.size(); i++) + { + CigarElement e = suppCigar.get(i); + CigarOperator op = e.getOperator(); + int len = e.getLength(); + + if(op == CigarOperator.M || op == CigarOperator.EQ || op == CigarOperator.X) + { + int mRefEnd = refCursor + len - 1; + prevMInsidePrimary = mRefEnd <= primaryRefEnd; + } + + if(op == CigarOperator.N && prevMInsidePrimary) + { + lastBadNIdx = i; + refAfterLastBadN = refCursor + len; // just after this N op + readAfterLastBadN = readCursor; + } + + if(op.consumesReferenceBases()) + { + refCursor += len; + } + if(op.consumesReadBases()) + { + readCursor += len; + } + } + + if(lastBadNIdx == -1 || refAfterLastBadN < 0) + { + return null; + } + return cutHeadAt(suppCigar, refAfterLastBadN, lastBadNIdx, readAfterLastBadN); + } + + private static ClampedSupp cutTailAt( + final List suppCigar, final int suppStart, + final int cutIdx, final int readBeforeCut) + { + List trimmed = new ArrayList<>(cutIdx + 1); + for(int i = 0; i < cutIdx; i++) + { + trimmed.add(suppCigar.get(i)); + } + int totalRead = CigarUtils.cigarBaseLength(suppCigar); + int trailingS = totalRead - readBeforeCut; + if(trailingS > 0) + { + trimmed.add(new CigarElement(trailingS, CigarOperator.S)); + } + return new ClampedSupp(suppStart, trimmed); + } + + private static ClampedSupp cutHeadAt( + final List suppCigar, final int newStart, + final int cutIdx, final int readBeforeCut) + { + List trimmed = new ArrayList<>(suppCigar.size() - cutIdx); + if(readBeforeCut > 0) + { + trimmed.add(new CigarElement(readBeforeCut, CigarOperator.S)); + } + for(int i = cutIdx + 1; i < suppCigar.size(); i++) + { + trimmed.add(suppCigar.get(i)); + } + return new ClampedSupp(newStart, trimmed); + } + + private static final class ClampedSupp + { + int Start; + List Cigar; + + ClampedSupp(final int start, final List cigar) + { + Start = start; + Cigar = cigar; + } + } + + private static final class Side + { + int Start; + List Cigar; + int LeadingS; + int TrailingS; + int LeadingM; + int TrailingM; + int RefEnd; + + private Side( + final int start, final List cigar, + final int leadingS, final int trailingS, + final int leadingM, final int trailingM, final int refEnd) + { + Start = start; + Cigar = cigar; + LeadingS = leadingS; + TrailingS = trailingS; + LeadingM = leadingM; + TrailingM = trailingM; + RefEnd = refEnd; + } + + static Side of(final int start, final List cigar) + { + return new Side(start, cigar, + CigarUtils.leftSoftClipLength(cigar), CigarUtils.rightSoftClipLength(cigar), + leadingMatchedRun(cigar), trailingMatchedRun(cigar), + start + CigarUtils.cigarAlignedLength(cigar) - 1); + } + } + + private static final class SnapPick + { + int L; + ChrBaseRegion Intron; + + SnapPick(final int l, final ChrBaseRegion intron) + { + L = l; + Intron = intron; + } + + static SnapPick miss() + { + return new SnapPick(-1, null); + } + } + + private static final class MergeOutcome + { + SupplementaryRejectReason Reject; + int MergedStart; + List MergedCigar; + ChrBaseRegion IntroducedIntron; + SupplementaryRecord MergedSupp; + boolean RightExtend; // which terminal softclip of the primary this merge resolved + + private MergeOutcome( + final SupplementaryRejectReason reject, + final int mergedStart, final List mergedCigar, + final ChrBaseRegion intron, final SupplementaryRecord supp, final boolean rightExtend) + { + Reject = reject; + MergedStart = mergedStart; + MergedCigar = mergedCigar; + IntroducedIntron = intron; + MergedSupp = supp; + RightExtend = rightExtend; + } + + static MergeOutcome reject(final SupplementaryRejectReason reason) + { + return new MergeOutcome(reason, -1, null, null, null, false); + } + + static MergeOutcome success( + final int start, final List cigar, + final ChrBaseRegion intron, final SupplementaryRecord supp, final boolean rightExtend) + { + return new MergeOutcome(null, start, cigar, intron, supp, rightExtend); + } + + boolean isSuccess() + { + return Reject == null; + } + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/supplementary/SupplementaryResult.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/supplementary/SupplementaryResult.java new file mode 100644 index 00000000000..b04df745e90 --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/supplementary/SupplementaryResult.java @@ -0,0 +1,26 @@ +package com.hartwig.hmftools.tars.liftback.supplementary; + +import java.util.Collections; +import java.util.List; + +import com.hartwig.hmftools.common.region.ChrBaseRegion; + +// Output DTO from SupplementaryResolver. On success, mergedCigar/mergedStart describe the new primary +// and droppedSupplementaryIndices lists absorbed supps. On failure, rejectReason carries the gate hit. +public record SupplementaryResult( + boolean merged, String mergedCigar, int mergedStart, List droppedSupplementaryIndices, + List introducedIntrons, int chainDepth, SupplementaryRejectReason rejectReason) +{ + private static final SupplementaryResult NO_MERGE_NO_OP = new SupplementaryResult( + false, null, -1, Collections.emptyList(), Collections.emptyList(), 0, + SupplementaryRejectReason.NO_TERMINAL_SOFTCLIP); + + public static SupplementaryResult noMerge(final SupplementaryRejectReason reason) + { + if(reason == SupplementaryRejectReason.NO_TERMINAL_SOFTCLIP) + { + return NO_MERGE_NO_OP; + } + return new SupplementaryResult(false, null, -1, Collections.emptyList(), Collections.emptyList(), 0, reason); + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/supplementary/SupplementaryStatistics.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/supplementary/SupplementaryStatistics.java new file mode 100644 index 00000000000..98d4e8686f9 --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/supplementary/SupplementaryStatistics.java @@ -0,0 +1,108 @@ +package com.hartwig.hmftools.tars.liftback.supplementary; + +import java.util.EnumMap; +import java.util.Map; + +// Per-decision counters for the supplementary resolver. Aggregated across the run and emitted into the +// liftback summary so we can see which gate is rejecting most candidates and how many chain +// merges happened. +public class SupplementaryStatistics +{ + private int mCandidatesEvaluated; + private int mMergedTotal; + private int mSuppClampApplied; + private final int[] mChainDepthCounts; // index = depth (1 = single merge, 2 = chain of two, ...) + private final Map mRejections; + + public SupplementaryStatistics(final int maxChainDepth) + { + mChainDepthCounts = new int[Math.max(maxChainDepth, 1) + 1]; + mRejections = new EnumMap<>(SupplementaryRejectReason.class); + } + + public void countCandidate() + { + ++mCandidatesEvaluated; + } + + public void countMergedChain(final int chainDepth) + { + ++mMergedTotal; + int slot = Math.min(chainDepth, mChainDepthCounts.length - 1); + ++mChainDepthCounts[slot]; + } + + public void countReject(final SupplementaryRejectReason reason) + { + mRejections.merge(reason, 1, Integer::sum); + } + + public void countSuppClamp() + { + ++mSuppClampApplied; + } + + public int suppClampApplied() + { + return mSuppClampApplied; + } + + public int candidatesEvaluated() + { + return mCandidatesEvaluated; + } + + public int mergedTotal() + { + return mMergedTotal; + } + + public int mergedAtChainDepth(final int depth) + { + if(depth <= 0 || depth >= mChainDepthCounts.length) + { + return 0; + } + return mChainDepthCounts[depth]; + } + + public int rejectCount(final SupplementaryRejectReason reason) + { + return mRejections.getOrDefault(reason, 0); + } + + // Snapshot / restore all counters so a discarded provisional mate decision can be rolled back without + // double-counting (see LiftBackGroupProcessor.processNameGroup). + public int[] snapshot() + { + SupplementaryRejectReason[] reasons = SupplementaryRejectReason.values(); + int[] snapshot = new int[3 + mChainDepthCounts.length + reasons.length]; + snapshot[0] = mCandidatesEvaluated; + snapshot[1] = mMergedTotal; + snapshot[2] = mSuppClampApplied; + System.arraycopy(mChainDepthCounts, 0, snapshot, 3, mChainDepthCounts.length); + for(int i = 0; i < reasons.length; ++i) + { + snapshot[3 + mChainDepthCounts.length + i] = mRejections.getOrDefault(reasons[i], 0); + } + return snapshot; + } + + public void restore(final int[] snapshot) + { + mCandidatesEvaluated = snapshot[0]; + mMergedTotal = snapshot[1]; + mSuppClampApplied = snapshot[2]; + System.arraycopy(snapshot, 3, mChainDepthCounts, 0, mChainDepthCounts.length); + SupplementaryRejectReason[] reasons = SupplementaryRejectReason.values(); + mRejections.clear(); + for(int i = 0; i < reasons.length; ++i) + { + int count = snapshot[3 + mChainDepthCounts.length + i]; + if(count != 0) + { + mRejections.put(reasons[i], count); + } + } + } +} diff --git a/tars/src/main/java/com/hartwig/hmftools/tars/liftback/supplementary/Tier.java b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/supplementary/Tier.java new file mode 100644 index 00000000000..fc8920599fc --- /dev/null +++ b/tars/src/main/java/com/hartwig/hmftools/tars/liftback/supplementary/Tier.java @@ -0,0 +1,11 @@ +package com.hartwig.hmftools.tars.liftback.supplementary; + +// Splice-motif tier of a candidate junction, ordered weakest to strongest. Declaration order is meaningful: +// callers rank candidates with compareTo (a stronger motif outranks a weaker one). +public enum Tier +{ + NONE, // neither donor nor acceptor matches a known splice motif + SEMI_CANONICAL, // GC-AG / AT-AC (and reverse-complement equivalents) + CANONICAL, // GT-AG (~99% of splice sites) + ANNOTATED // matches an annotated junction from the sidecar +} diff --git a/tars/src/main/resources/log4j2.xml b/tars/src/main/resources/log4j2.xml new file mode 100644 index 00000000000..302549adf7d --- /dev/null +++ b/tars/src/main/resources/log4j2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/tars/src/main/resources/tars.version b/tars/src/main/resources/tars.version new file mode 100644 index 00000000000..686c865d141 --- /dev/null +++ b/tars/src/main/resources/tars.version @@ -0,0 +1,2 @@ +version=${project.version} +build.date=${timestamp} diff --git a/tars/src/test/java/com/hartwig/hmftools/tars/common/ContigSidecarTest.java b/tars/src/test/java/com/hartwig/hmftools/tars/common/ContigSidecarTest.java new file mode 100644 index 00000000000..11dea844826 --- /dev/null +++ b/tars/src/test/java/com/hartwig/hmftools/tars/common/ContigSidecarTest.java @@ -0,0 +1,81 @@ +package com.hartwig.hmftools.tars.common; + +import static com.hartwig.hmftools.common.test.GeneTestUtils.CHR_1; +import static com.hartwig.hmftools.common.test.GeneTestUtils.CHR_2; +import static com.hartwig.hmftools.common.utils.file.FileDelimiters.TSV_DELIM; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import com.hartwig.hmftools.common.region.BaseRegion; + +import org.junit.Test; + +public class ContigSidecarTest +{ + @Test + public void testRoundTrip() throws IOException + { + File tempFile = Files.createTempFile("contig_sidecar_", ".tsv").toFile(); + tempFile.deleteOnExit(); + + List entries = List.of( + new ContigEntry( + "chr1_tx", 1, 200, "ENSG0000001", "GENEA", "ENST0000001", CHR_1, 1, + List.of(new BaseRegion(100, 199), new BaseRegion(300, 399))), + new ContigEntry( + "chr2_tx", 1, 51, "ENSG0000002", "GENEB", "ENST0000002", CHR_2, -1, + List.of(new BaseRegion(1000, 1050)))); + + ContigSidecar.write(tempFile.getAbsolutePath(), entries); + + List readBack = ContigSidecar.read(tempFile.getAbsolutePath()); + + assertEquals(2, readBack.size()); + + ContigEntry first = readBack.get(0); + assertEquals("chr1_tx", first.contigName()); + assertEquals(1, first.altStart()); + assertEquals(200, first.altEnd()); + assertEquals("ENSG0000001", first.geneId()); + assertEquals("GENEA", first.geneName()); + assertEquals("ENST0000001", first.transName()); + assertEquals(CHR_1, first.chromosome()); + assertEquals(1, first.strand()); + assertEquals(2, first.exonSpans().size()); + assertEquals(new BaseRegion(100, 199), first.exonSpans().get(0)); + assertEquals(new BaseRegion(300, 399), first.exonSpans().get(1)); + + ContigEntry second = readBack.get(1); + assertEquals("chr2_tx", second.contigName()); + assertEquals(1, second.altStart()); + assertEquals(51, second.altEnd()); + assertEquals("ENST0000002", second.transName()); + assertEquals(-1, second.strand()); + assertEquals(1, second.exonSpans().size()); + assertEquals(new BaseRegion(1000, 1050), second.exonSpans().get(0)); + } + + @Test + public void testHeaderOnlySidecarReadsAsEmpty() throws IOException + { + // a sidecar with only its header row (no entries) loads as an empty list rather than failing + File tempFile = Files.createTempFile("contig_sidecar_empty_", ".tsv").toFile(); + tempFile.deleteOnExit(); + + String header = Arrays.stream(ContigSidecar.Column.values()) + .map(Enum::name) + .collect(Collectors.joining(TSV_DELIM)); + Files.writeString(tempFile.toPath(), header + "\n"); + + List readBack = ContigSidecar.read(tempFile.getAbsolutePath()); + assertTrue(readBack.isEmpty()); + } +} diff --git a/tars/src/test/java/com/hartwig/hmftools/tars/fasta/AltContigPackerTest.java b/tars/src/test/java/com/hartwig/hmftools/tars/fasta/AltContigPackerTest.java new file mode 100644 index 00000000000..d76603e8308 --- /dev/null +++ b/tars/src/test/java/com/hartwig/hmftools/tars/fasta/AltContigPackerTest.java @@ -0,0 +1,106 @@ +package com.hartwig.hmftools.tars.fasta; + +import static com.hartwig.hmftools.common.test.GeneTestUtils.CHR_1; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.List; + +import com.hartwig.hmftools.common.region.BaseRegion; +import com.hartwig.hmftools.tars.common.ContigEntry; + +import org.junit.Test; + +public class AltContigPackerTest +{ + private static final int SPACER = 5; + + private static TranscriptContigBuilder.TranscriptContigResult txResult( + final String transName, final String sequence, final List exonSpans) + { + return new TranscriptContigBuilder.TranscriptContigResult( + "G_" + transName, "GENE_" + transName, transName, CHR_1, 1, exonSpans, sequence); + } + + @Test + public void testEmptyInputProducesEmptyResult() + { + AltContigPacker packer = new AltContigPacker(SPACER); + AltContigPacker.PackResult result = packer.pack(CHR_1, List.of()); + + assertEquals("1_tx", result.altContig()); + assertEquals("", result.sequence()); + assertTrue(result.entries().isEmpty()); + assertTrue(result.isEmpty()); + } + + @Test + public void testSingleTranscriptHasNoSpacer() + { + AltContigPacker packer = new AltContigPacker(SPACER); + AltContigPacker.PackResult result = packer.pack( + CHR_1, List.of(txResult("T1", "ACGTACGT", List.of(new BaseRegion(100, 107))))); + + assertEquals("ACGTACGT", result.sequence()); + assertEquals(1, result.entries().size()); + + ContigEntry entry = result.entries().get(0); + assertEquals("1_tx", entry.contigName()); + assertEquals(1, entry.altStart()); + assertEquals(8, entry.altEnd()); + assertEquals("T1", entry.transName()); + } + + @Test + public void testTwoTranscriptsAreSeparatedBySpacer() + { + AltContigPacker packer = new AltContigPacker(SPACER); + AltContigPacker.PackResult result = packer.pack(CHR_1, List.of( + txResult("T1", "AAAA", List.of(new BaseRegion(100, 103))), + txResult("T2", "CCC", List.of(new BaseRegion(200, 202))))); + + assertEquals("AAAA" + "NNNNN" + "CCC", result.sequence()); + assertEquals(2, result.entries().size()); + + ContigEntry first = result.entries().get(0); + assertEquals(1, first.altStart()); + assertEquals(4, first.altEnd()); + + ContigEntry second = result.entries().get(1); + assertEquals(10, second.altStart()); + assertEquals(12, second.altEnd()); + } + + @Test + public void testThreeTranscriptsHaveContiguousNonOverlappingRanges() + { + AltContigPacker packer = new AltContigPacker(SPACER); + AltContigPacker.PackResult result = packer.pack(CHR_1, List.of( + txResult("T1", "AAAAAA", List.of(new BaseRegion(100, 105))), + txResult("T2", "CCC", List.of(new BaseRegion(200, 202))), + txResult("T3", "GGGGGGGG", List.of(new BaseRegion(300, 307))))); + + assertEquals(6 + 5 + 3 + 5 + 8, result.sequence().length()); + for(int i = 0; i < result.entries().size() - 1; ++i) + { + ContigEntry a = result.entries().get(i); + ContigEntry b = result.entries().get(i + 1); + assertEquals(a.altEnd() + SPACER + 1, b.altStart()); + } + } + + @Test + public void testEntryMetadataIsCarriedThrough() + { + AltContigPacker packer = new AltContigPacker(SPACER); + AltContigPacker.PackResult result = packer.pack( + CHR_1, List.of(txResult("T1", "ACGT", List.of(new BaseRegion(100, 103))))); + + ContigEntry entry = result.entries().get(0); + assertEquals("G_T1", entry.geneId()); + assertEquals("GENE_T1", entry.geneName()); + assertEquals(CHR_1, entry.chromosome()); + assertEquals(List.of(new BaseRegion(100, 103)), entry.exonSpans()); + } +} diff --git a/tars/src/test/java/com/hartwig/hmftools/tars/fasta/TranscriptContigBuilderTest.java b/tars/src/test/java/com/hartwig/hmftools/tars/fasta/TranscriptContigBuilderTest.java new file mode 100644 index 00000000000..0ba1a8a617c --- /dev/null +++ b/tars/src/test/java/com/hartwig/hmftools/tars/fasta/TranscriptContigBuilderTest.java @@ -0,0 +1,176 @@ +package com.hartwig.hmftools.tars.fasta; + +import static com.hartwig.hmftools.common.genome.region.Strand.NEG_STRAND; +import static com.hartwig.hmftools.common.genome.region.Strand.POS_STRAND; +import static com.hartwig.hmftools.common.test.GeneTestUtils.CHR_1; +import static com.hartwig.hmftools.common.test.GeneTestUtils.createEnsemblGeneData; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +import com.hartwig.hmftools.common.gene.ExonData; +import com.hartwig.hmftools.common.gene.GeneData; +import com.hartwig.hmftools.common.gene.TranscriptData; +import com.hartwig.hmftools.common.region.BaseRegion; +import com.hartwig.hmftools.common.test.MockRefGenome; + +import org.junit.Test; + +public class TranscriptContigBuilderTest +{ + private static final String GENE_ID = "ENSG00000171862"; + private static final String GENE_NAME = "PTEN"; + private static final String TRANS_NAME = "ENST00000399770"; + + @Test + public void testThreeExonForwardStrandTranscript() + { + TranscriptData transcript = transcript(); + addExon(transcript, 100, 199, 1); + addExon(transcript, 300, 399, 2); + addExon(transcript, 500, 549, 3); + + TranscriptContigBuilder builder = new TranscriptContigBuilder(buildMockRef()); + TranscriptContigBuilder.TranscriptContigResult result = builder.build(gene(POS_STRAND), transcript); + + assertNotNull(result); + assertEquals(GENE_ID, result.geneId()); + assertEquals(GENE_NAME, result.geneName()); + assertEquals(TRANS_NAME, result.transName()); + assertEquals(CHR_1, result.chromosome()); + + assertEquals(3, result.exonSpans().size()); + assertEquals(new BaseRegion(100, 199), result.exonSpans().get(0)); + assertEquals(new BaseRegion(300, 399), result.exonSpans().get(1)); + assertEquals(new BaseRegion(500, 549), result.exonSpans().get(2)); + + assertEquals(100 + 100 + 50, result.sequence().length()); + } + + @Test + public void testNegativeStrandTranscriptStoresExonsInGenomicOrder() + { + // Rank-order for a negative-strand transcript is genomic-descending; builder must re-sort to genomic-ascending. + TranscriptData transcript = transcript(); + addExon(transcript, 500, 549, 1); // 5' end of mRNA, but genomic-high + addExon(transcript, 300, 399, 2); + addExon(transcript, 100, 199, 3); + + TranscriptContigBuilder builder = new TranscriptContigBuilder(buildMockRef()); + TranscriptContigBuilder.TranscriptContigResult result = builder.build(gene(NEG_STRAND), transcript); + + assertNotNull(result); + assertEquals(100, result.exonSpans().get(0).start()); + assertEquals(300, result.exonSpans().get(1).start()); + assertEquals(500, result.exonSpans().get(2).start()); + } + + @Test + public void testEmittedSequenceMatchesExonConcatenation() + { + TranscriptData transcript = transcript(); + addExon(transcript, 100, 199, 1); + addExon(transcript, 300, 399, 2); + addExon(transcript, 500, 549, 3); + + MockRefGenome ref = buildMockRef(); + TranscriptContigBuilder builder = new TranscriptContigBuilder(ref); + TranscriptContigBuilder.TranscriptContigResult result = builder.build(gene(POS_STRAND), transcript); + + assertNotNull(result); + + String expected = ref.getBaseString(CHR_1, 100, 199) + + ref.getBaseString(CHR_1, 300, 399) + + ref.getBaseString(CHR_1, 500, 549); + assertEquals(expected, result.sequence()); + } + + @Test + public void testMultipleTranscriptsEachGetTheirOwnContig() + { + TranscriptData transA = transcript(1, "ENST00000001", true); + addExon(transA, 100, 199, 1); + addExon(transA, 300, 399, 2); + + TranscriptData transB = transcript(2, "ENST00000002", false); + addExon(transB, 100, 199, 1); + addExon(transB, 500, 549, 2); + + TranscriptContigBuilder builder = new TranscriptContigBuilder(buildMockRef()); + TranscriptContigBuilder.TranscriptContigResult resultA = builder.build(gene(POS_STRAND), transA); + TranscriptContigBuilder.TranscriptContigResult resultB = builder.build(gene(POS_STRAND), transB); + + assertNotNull(resultA); + assertNotNull(resultB); + + assertEquals("ENST00000001", resultA.transName()); + assertEquals("ENST00000002", resultB.transName()); + + assertEquals(2, resultA.exonSpans().size()); + assertEquals(2, resultB.exonSpans().size()); + + assertEquals(200, resultA.sequence().length()); + assertEquals(150, resultB.sequence().length()); + } + + @Test + public void testTranscriptWithNoExonsReturnsNull() + { + TranscriptData empty = transcript(); + TranscriptContigBuilder builder = new TranscriptContigBuilder(buildMockRef()); + assertNull(builder.build(gene(POS_STRAND), empty)); + } + + @Test + public void testSingleExonTranscriptHasNoIntrons() + { + TranscriptData transcript = transcript(); + addExon(transcript, 100, 200, 1); + + TranscriptContigBuilder builder = new TranscriptContigBuilder(buildMockRef()); + TranscriptContigBuilder.TranscriptContigResult result = builder.build(gene(POS_STRAND), transcript); + + assertNotNull(result); + assertEquals(1, result.exonSpans().size()); + assertEquals(101, result.sequence().length()); + } + + private static GeneData gene(final byte strand) + { + return createEnsemblGeneData(GENE_ID, GENE_NAME, CHR_1, strand, 100, 549); + } + + // Local builders are retained because GeneTestUtils.createTransExons only models uniform exon length and + // genomic-ascending rank, whereas these tests need non-uniform exons (100/100/50bp) and a negative-strand + // transcript whose ranks descend genomically. + private static TranscriptData transcript() + { + return transcript(1, TRANS_NAME, true); + } + + private static TranscriptData transcript(final int transId, final String transName, final boolean canonical) + { + return new TranscriptData(transId, transName, GENE_ID, canonical, POS_STRAND, 0, 0, null, null, "protein_coding", null); + } + + private static void addExon(final TranscriptData transcript, final int start, final int end, final int rank) + { + transcript.exons().add(new ExonData(transcript.TransId, start, end, rank, -1, -1)); + } + + private static MockRefGenome buildMockRef() + { + // position-distinguishable bases so sequence-content assertions actually catch bad concatenations + MockRefGenome refGenome = new MockRefGenome(true); + char[] alphabet = { 'A', 'C', 'G', 'T' }; + StringBuilder bases = new StringBuilder(); + bases.append('N'); // pad index 0 + for(int i = 1; i <= 1000; ++i) + { + bases.append(alphabet[i % 4]); + } + refGenome.RefGenomeMap.put(CHR_1, bases.toString()); + return refGenome; + } +} diff --git a/tars/src/test/java/com/hartwig/hmftools/tars/liftback/AlignmentScorerTest.java b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/AlignmentScorerTest.java new file mode 100644 index 00000000000..9408f205e62 --- /dev/null +++ b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/AlignmentScorerTest.java @@ -0,0 +1,73 @@ +package com.hartwig.hmftools.tars.liftback; + +import static org.junit.Assert.assertEquals; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; + +import com.hartwig.hmftools.tars.liftback.supplementary.RefSequenceSource; + +import org.junit.Test; + +public class AlignmentScorerTest +{ + private static final String CHR = "chr1"; + + // reference that returns all-'A' bases for any requested range. + private static final RefSequenceSource ALL_A = (chromosome, posStart, posEnd) -> + { + byte[] bases = new byte[posEnd - posStart + 1]; + Arrays.fill(bases, (byte) 'A'); + return bases; + }; + + private static byte[] read(final String sequence) + { + return sequence.getBytes(StandardCharsets.US_ASCII); + } + + private static int score(final String cigar, final byte[] readBases) + { + return AlignmentScorer.score(ALL_A, CHR, 1, cigar, readBases); + } + + @Test + public void testFullMatchScoresOnePerBase() + { + assertEquals(10, score("10M", read("AAAAAAAAAA"))); + } + + @Test + public void testMismatchCostsFour() + { + // one C against an all-A reference: 9 matches (+9), 1 mismatch (-4) = 5. + assertEquals(5, score("10M", read("AAAACAAAAA"))); + } + + @Test + public void testDeletionIsAffineGap() + { + // 10 matched bases (+10) and a 2bp deletion (GAP_OPEN -6, GAP_EXTEND -1) = 3. + assertEquals(3, score("5M2D5M", read("AAAAAAAAAA"))); + } + + @Test + public void testIntronIsFree() + { + // the N contributes nothing; only the 10 matched bases score. + assertEquals(10, score("5M100N5M", read("AAAAAAAAAA"))); + } + + @Test + public void testSoftClipIsFree() + { + // 3 clipped bases contribute nothing; the 7 aligned bases score +7. + assertEquals(7, score("3S7M", read("AAAAAAAAAA"))); + } + + @Test + public void testNoRefSourceReturnsSentinel() + { + assertEquals(Integer.MIN_VALUE, AlignmentScorer.score(null, CHR, 1, "10M", read("AAAAAAAAAA"))); + } +} diff --git a/tars/src/test/java/com/hartwig/hmftools/tars/liftback/ChunkProducerTest.java b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/ChunkProducerTest.java new file mode 100644 index 00000000000..2f22d1527bd --- /dev/null +++ b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/ChunkProducerTest.java @@ -0,0 +1,93 @@ +package com.hartwig.hmftools.tars.liftback; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.Test; + +import htsjdk.samtools.SAMFileHeader; +import htsjdk.samtools.SAMRecord; + +// the producer's one safety-critical invariant: a chunk is only cut at a read-name boundary, so no +// fragment's records are ever split across two chunks (a split fragment would defeat the per-group cache +// and orphan mates/supps for dedup). +public class ChunkProducerTest +{ + private static SAMRecord read(final String name) + { + SAMRecord record = new SAMRecord(new SAMFileHeader()); + record.setReadName(name); + return record; + } + + private static List> chunk(final List records, final int targetReads) + throws InterruptedException + { + List> chunks = new ArrayList<>(); + ChunkProducer.streamChunks(records.iterator(), targetReads, chunks::add, null); + return chunks; + } + + private static void assertWholeFragments(final List> chunks) + { + for(int i = 0; i + 1 < chunks.size(); ++i) + { + String lastName = chunks.get(i).get(chunks.get(i).size() - 1).getReadName(); + String nextName = chunks.get(i + 1).get(0).getReadName(); + assertTrue("fragment split across chunk boundary: " + lastName, !lastName.equals(nextName)); + } + } + + @Test + public void cutsOnlyAtNameBoundary() throws InterruptedException + { + // three records per fragment, target 2 reads: the cut must wait for the name to change rather than + // firing mid-fragment, so each chunk holds whole fragments. + List records = new ArrayList<>(); + for(int fragment = 0; fragment < 5; ++fragment) + { + records.add(read("frag" + fragment)); + records.add(read("frag" + fragment)); + records.add(read("frag" + fragment)); + } + + List> chunks = chunk(records, 2); + + assertWholeFragments(chunks); + + int total = 0; + for(final List chunk : chunks) + { + total += chunk.size(); + } + assertEquals(15, total); + } + + @Test + public void emitsSingleChunkUnderTarget() throws InterruptedException + { + List records = List.of(read("a"), read("a"), read("b")); + List> chunks = chunk(records, 5000); + assertEquals(1, chunks.size()); + assertEquals(3, chunks.get(0).size()); + } + + @Test + public void oversizeFragmentStaysWhole() throws InterruptedException + { + // a single fragment larger than the target must not be split -- the boundary check gates the cut. + List records = new ArrayList<>(); + for(int i = 0; i < 10; ++i) + { + records.add(read("big")); + } + records.add(read("next")); + + List> chunks = chunk(records, 3); + assertWholeFragments(chunks); + assertEquals(10, chunks.get(0).size()); + } +} diff --git a/tars/src/test/java/com/hartwig/hmftools/tars/liftback/ContigTranslatorTest.java b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/ContigTranslatorTest.java new file mode 100644 index 00000000000..d0cddf16fd9 --- /dev/null +++ b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/ContigTranslatorTest.java @@ -0,0 +1,357 @@ +package com.hartwig.hmftools.tars.liftback; + +import static com.hartwig.hmftools.common.test.GeneTestUtils.CHR_1; +import static com.hartwig.hmftools.tars.liftback.TarsTestFixtures.threeExonContig; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.util.List; + +import com.hartwig.hmftools.common.region.BaseRegion; +import com.hartwig.hmftools.tars.common.ContigEntry; + +import org.junit.Test; + +import htsjdk.samtools.Cigar; +import htsjdk.samtools.CigarElement; +import htsjdk.samtools.TextCigarCodec; + +// All tests use TarsTestFixtures.threeExonContig(), whose packed contig coordinates map to the genome as: +// +// contig 1..100 -> exon1 chr1 100..199 (100 bp) +// contig 101..200 -> exon2 chr1 300..399 (100 bp) +// contig 201..250 -> exon3 chr1 500..549 ( 50 bp) +// introns (implied as N on lift): 200..299 (100 bp) and 400..499 (100 bp) +// +// So a contig position lifts to genomic = exon.start + (contigPos - exon.contigStart). E.g. contig 51 -> 150 +// (exon1), contig 91 -> 190 (exon1), contig 201 -> 500 (exon3), contig 250 -> 549 (last base). +public class ContigTranslatorTest +{ + @Test + public void testReadEntirelyInFirstExon() + { + // contig 1..50 sits inside exon1 -> chr1 100, no boundary crossed. + ContigTranslator.TranslationResult result = ContigTranslator.translate(threeExonContig(), 1, cigar("50M")); + + assertNotNull(result); + assertEquals(CHR_1, result.chromosome()); + assertEquals(100, result.genomicStart()); + assertEquals("50M", result.genomicCigar().toString()); + assertTrue(result.impliedIntrons().isEmpty()); + } + + @Test + public void testReadEntirelyInLastExon() + { + // contig 201..230 sits inside exon3 -> chr1 500, no boundary crossed. + ContigTranslator.TranslationResult result = ContigTranslator.translate(threeExonContig(), 201, cigar("30M")); + + assertNotNull(result); + assertEquals(500, result.genomicStart()); + assertEquals("30M", result.genomicCigar().toString()); + assertTrue(result.impliedIntrons().isEmpty()); + } + + @Test + public void testReadCrossingOneJunction() + { + // contig 51..150 spans the exon1/exon2 boundary (contig 100|101): 50 bp of exon1 (from chr1 150), + // then intron1 as 100N, then 50 bp of exon2. + ContigTranslator.TranslationResult result = ContigTranslator.translate(threeExonContig(), 51, cigar("100M")); + + assertNotNull(result); + assertEquals(150, result.genomicStart()); + assertEquals("50M100N50M", result.genomicCigar().toString()); + assertEquals(1, result.impliedIntrons().size()); + assertEquals(new BaseRegion(200, 299), result.impliedIntrons().get(0)); + } + + @Test + public void testReadCrossingTwoJunctions() + { + // contig 91..220 spans both boundaries: 10 bp tail of exon1 (from chr1 190), intron1 (100N), all + // 100 bp of exon2, intron2 (100N), 20 bp head of exon3. + ContigTranslator.TranslationResult result = ContigTranslator.translate(threeExonContig(), 91, cigar("130M")); + + assertNotNull(result); + assertEquals(190, result.genomicStart()); + assertEquals("10M100N100M100N20M", result.genomicCigar().toString()); + assertEquals(2, result.impliedIntrons().size()); + assertEquals(new BaseRegion(200, 299), result.impliedIntrons().get(0)); + assertEquals(new BaseRegion(400, 499), result.impliedIntrons().get(1)); + } + + @Test + public void testReadWithInsertion() + { + ContigTranslator.TranslationResult result = ContigTranslator.translate(threeExonContig(), 1, cigar("30M5I20M")); + + assertNotNull(result); + assertEquals(100, result.genomicStart()); + assertEquals("30M5I20M", result.genomicCigar().toString()); + assertTrue(result.impliedIntrons().isEmpty()); + } + + @Test + public void testReadWithDeletion() + { + ContigTranslator.TranslationResult result = ContigTranslator.translate(threeExonContig(), 1, cigar("30M5D20M")); + + assertNotNull(result); + assertEquals(100, result.genomicStart()); + assertEquals("30M5D20M", result.genomicCigar().toString()); + assertTrue(result.impliedIntrons().isEmpty()); + } + + @Test + public void testReadExactlyFillingFirstExon() + { + // fills exon 1 exactly - guards against a spurious trailing N + ContigTranslator.TranslationResult result = ContigTranslator.translate(threeExonContig(), 1, cigar("100M")); + + assertNotNull(result); + assertEquals(100, result.genomicStart()); + assertEquals("100M", result.genomicCigar().toString()); + assertTrue(result.impliedIntrons().isEmpty()); + } + + @Test + public void testReadExtendingPastLastSpanIsClampedToTrailingSoftClip() + { + // contig 231 -> exon3 chr1 530; only 20 bp remain to the contig end (250), so 20M then the 30 bp + // overhang past the last span becomes a trailing soft-clip. + ContigTranslator.TranslationResult result = ContigTranslator.translate(threeExonContig(), 231, cigar("50M")); + + assertNotNull(result); + assertEquals(530, result.genomicStart()); + assertEquals("20M30S", result.genomicCigar().toString()); + } + + @Test + public void testContigPosBeyondContigLengthReturnsNull() + { + // contig 251 is past the contig end (length 250) -> untranslatable. + assertNull(ContigTranslator.translate(threeExonContig(), 251, cigar("10M"))); + } + + @Test + public void testLeadingOverhangClampedToSoftClip() + { + // pos 0 is 1 base before altStart; leading M absorbs into soft-clip. + ContigTranslator.TranslationResult result = ContigTranslator.translate(threeExonContig(), 0, cigar("10M")); + + assertNotNull(result); + assertEquals(100, result.genomicStart()); + assertEquals("1S9M", result.genomicCigar().toString()); + } + + @Test + public void testLeadingOverhangExceedsLeadingMReturnsNull() + { + // contig start -5 is 6 bases before the contig (pos 1), but only 4M lead it -> overhang can't be + // absorbed into a soft-clip, so the lift fails. + assertNull(ContigTranslator.translate(threeExonContig(), -5, cigar("4M10S"))); + } + + @Test + public void testReadAtLastBaseOfContig() + { + // contig 250 is the final base of exon3 -> chr1 549. + ContigTranslator.TranslationResult result = ContigTranslator.translate(threeExonContig(), 250, cigar("1M")); + + assertNotNull(result); + assertEquals(549, result.genomicStart()); + assertEquals("1M", result.genomicCigar().toString()); + } + + @Test + public void testReadWithLeadingAndTrailingSoftClips() + { + // softclips consume query only - genomic start is pos-based, not query-based; both clips survive the lift. + ContigTranslator.TranslationResult result = ContigTranslator.translate(threeExonContig(), 51, cigar("10S100M10S")); + + assertNotNull(result); + assertEquals(150, result.genomicStart()); + assertEquals("10S50M100N50M10S", result.genomicCigar().toString()); + } + + @Test + public void testDeletionCrossingJunctionAboveThreshold() + { + // contig 91 -> chr1 190; the 10M reaches the exon1 end (contig 100), so the D sits right at the + // boundary. 10D exceeds SPLICE_FLANKING_DELETION_MAX_BP (5) so it is preserved after the intron N, not absorbed. + ContigTranslator.TranslationResult result = ContigTranslator.translate(threeExonContig(), 91, cigar("10M10D5M")); + + assertNotNull(result); + assertEquals(190, result.genomicStart()); + assertEquals("10M100N10D5M", result.genomicCigar().toString()); + assertEquals(1, result.impliedIntrons().size()); + } + + @Test + public void testDeletionCrossingJunctionCollapsedIntoSplice() + { + // tx-FASTA off-by-N artefact: 5D (at SPLICE_FLANKING_DELETION_MAX_BP = 5) straddling an exon boundary is absorbed into the splice N. + ContigTranslator.TranslationResult result = ContigTranslator.translate(threeExonContig(), 91, cigar("10M5D5M")); + + assertNotNull(result); + assertEquals(190, result.genomicStart()); + assertEquals("10M105N5M", result.genomicCigar().toString()); + assertEquals(1, result.impliedIntrons().size()); + } + + @Test + public void testDeletionsBothSidesOfJunctionCollapsedIntoSplice() + { + // D straddles the boundary: lift splits it, then both flanking Ds are absorbed into the N. + ContigTranslator.TranslationResult result = ContigTranslator.translate(threeExonContig(), 92, cigar("9M3D6M")); + + assertNotNull(result); + assertEquals(191, result.genomicStart()); + assertEquals("9M103N6M", result.genomicCigar().toString()); + assertEquals(1, result.impliedIntrons().size()); + } + + @Test + public void testTwoExonContigBoundaryCases() + { + // a different two-exon contig: contig 1..50 -> exon1 chr1 100..149, contig 51..100 -> exon2 chr1 200..249; + // implied intron 150..199. + ContigEntry twoExon = new ContigEntry( + "ensG_X_T", 1, 100, "G", "X", "T", CHR_1, 1, + List.of(new BaseRegion(100, 149), new BaseRegion(200, 249))); + + // contig 1..50 fills exon1 exactly -> chr1 100, no spurious trailing N. + ContigTranslator.TranslationResult result = ContigTranslator.translate(twoExon, 1, cigar("50M")); + assertNotNull(result); + assertEquals(100, result.genomicStart()); + assertEquals("50M", result.genomicCigar().toString()); + + // contig 50..100 crosses the boundary: 1 bp tail of exon1 (chr1 149), 50N intron, 50 bp of exon2. + result = ContigTranslator.translate(twoExon, 50, cigar("51M")); + assertNotNull(result); + assertEquals(149, result.genomicStart()); + assertEquals("1M50N50M", result.genomicCigar().toString()); + } + + @Test + public void testHasSoftClipAtExonBoundary() + { + // softclip abutting an interior exon boundary, trailing then leading -> true + assertTrue(ContigTranslator.hasSoftClipAtExonBoundary(threeExonContig(), 71, cigar("30M20S"))); + assertTrue(ContigTranslator.hasSoftClipAtExonBoundary(threeExonContig(), 101, cigar("20S30M"))); + + // softclip away from any boundary, and no softclip at all -> false + assertFalse(ContigTranslator.hasSoftClipAtExonBoundary(threeExonContig(), 60, cigar("30M20S"))); + assertFalse(ContigTranslator.hasSoftClipAtExonBoundary(threeExonContig(), 71, cigar("30M"))); + + // single-exon contig has no interior boundaries -> false either side + ContigEntry singleExon = new ContigEntry( + "single", 1, 100, "G", "X", "T", CHR_1, 1, List.of(new BaseRegion(100, 199))); + assertFalse(ContigTranslator.hasSoftClipAtExonBoundary(singleExon, 1, cigar("50S50M"))); + assertFalse(ContigTranslator.hasSoftClipAtExonBoundary(singleExon, 1, cigar("50M50S"))); + + // outer edges of the contig are not interior boundaries -> false + assertFalse(ContigTranslator.hasSoftClipAtExonBoundary(threeExonContig(), 1, cigar("20S30M"))); + assertFalse(ContigTranslator.hasSoftClipAtExonBoundary(threeExonContig(), 201, cigar("50M20S"))); + } + + @Test + public void testTrimTrailingMicroAnchor() + { + // 3M tail at threshold 2 -> kept (trim fires only strictly below threshold) + assertEquals("100M500N3M48S", ContigTranslator.trimTrailingMicroAnchor(cigar("100M500N3M48S"), 2).toString()); + + // 2M tail below threshold 3 -> dropped, intron collapsed into the trailing clip + assertEquals("100M51S", ContigTranslator.trimTrailingMicroAnchor(cigar("100M500N2M49S"), 3).toString()); + + // 3M tail at threshold 3 -> kept + assertEquals("100M500N3M48S", ContigTranslator.trimTrailingMicroAnchor(cigar("100M500N3M48S"), 3).toString()); + + // 1bp tail -> dropped + assertEquals("80M71S", ContigTranslator.trimTrailingMicroAnchor(cigar("80M2000N1M70S"), 3).toString()); + + // no trailing softclip to fold into -> unchanged + assertEquals("80M2000N2M", ContigTranslator.trimTrailingMicroAnchor(cigar("80M2000N2M"), 3).toString()); + + // no preceding N -> nothing to trim; adjacent M ops are not a junction + assertEquals("100M", ContigTranslator.trimTrailingMicroAnchor(cigar("100M"), 3).toString()); + assertEquals("80M3M48S", ContigTranslator.trimTrailingMicroAnchor(cigar("80M3M48S"), 3).toString()); + + // no preceding M anchor - trimming would leave only softclips, so refuse + assertEquals("50S100N2M99S", ContigTranslator.trimTrailingMicroAnchor(cigar("50S100N2M99S"), 3).toString()); + + // 4M tail above threshold -> kept + assertEquals("100M500N4M47S", ContigTranslator.trimTrailingMicroAnchor(cigar("100M500N4M47S"), 3).toString()); + } + + @Test + public void testTrimMicroAnchors() + { + // leading 2M anchor dropped, rolled into leading S; start advances by anchor + intron = 3715 + ContigTranslator.MicroAnchorResult out = ContigTranslator.trimMicroAnchors(cigar("38S2M3713N68M4I39M"), 1, 3); + assertEquals("40S68M4I39M", out.AdjustedCigar.toString()); + assertEquals(3715, out.StartShift); + + // leading 3M anchor at threshold -> kept + out = ContigTranslator.trimMicroAnchors(cigar("38S3M3713N68M"), 1, 3); + assertEquals("38S3M3713N68M", out.AdjustedCigar.toString()); + assertEquals(0, out.StartShift); + + // both ends trimmed + out = ContigTranslator.trimMicroAnchors(cigar("10S2M500N80M600N1M20S"), 1, 3); + assertEquals("12S80M21S", out.AdjustedCigar.toString()); + assertEquals(502, out.StartShift); + + // nothing tiny -> unchanged + out = ContigTranslator.trimMicroAnchors(cigar("38S68M3713N40M"), 1, 3); + assertEquals("38S68M3713N40M", out.AdjustedCigar.toString()); + assertEquals(0, out.StartShift); + + // bare leading anchor (no softclip) means the read genuinely starts mid-exon -> keep it + out = ContigTranslator.trimMicroAnchors(cigar("2M80N149M"), 1, 3); + assertEquals("2M80N149M", out.AdjustedCigar.toString()); + assertEquals(0, out.StartShift); + + // same 2bp anchor but with trailing softclip: bwa over-ran the boundary, junction unsupported -> drop it + out = ContigTranslator.trimMicroAnchors(cigar("117M89N2M32S"), 1, 3); + assertEquals("117M34S", out.AdjustedCigar.toString()); + assertEquals(0, out.StartShift); + } + + @Test + public void testContiguousSpansEmitNoZeroLengthIntron() + { + // adjacent spans (no gap) imply no intron, so no 0-length N element is emitted + ContigEntry contiguous = new ContigEntry( + "ensG_X_T", 1, 100, "G", "X", "T", CHR_1, 1, + List.of(new BaseRegion(100, 149), new BaseRegion(150, 199))); + + ContigTranslator.TranslationResult result = ContigTranslator.translate(contiguous, 1, cigar("100M")); + + assertNotNull(result); + assertEquals(100, result.genomicStart()); + assertFalse(result.genomicCigar().getCigarElements().stream().anyMatch(x -> x.getLength() == 0)); + assertEquals("100M", result.genomicCigar().toString()); + assertTrue(result.impliedIntrons().isEmpty()); + } + + @Test + public void testDropZeroLengthElementsAndMergeFlankingIntrons() + { + // a 0-length interior element (eg a 0M for a zero-span exon between two introns) is dropped, then the + // flanking introns merge into one + List elements = cigar("72M102N0M2538N79M").getCigarElements(); + Cigar result = new Cigar(ContigTranslator.mergeAdjacentSameOp(ContigTranslator.dropZeroLength(elements))); + assertEquals("72M2640N79M", result.toString()); + } + + private static Cigar cigar(final String cigarString) + { + return TextCigarCodec.decode(cigarString); + } +} diff --git a/tars/src/test/java/com/hartwig/hmftools/tars/liftback/ExcludedRegionsTest.java b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/ExcludedRegionsTest.java new file mode 100644 index 00000000000..3cdb080f106 --- /dev/null +++ b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/ExcludedRegionsTest.java @@ -0,0 +1,45 @@ +package com.hartwig.hmftools.tars.liftback; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.hartwig.hmftools.common.region.ChrBaseRegion; + +import org.junit.Test; + +// excludes() overlap contract, queried post-lift on lifted genomic coords. +public class ExcludedRegionsTest +{ + private static final String CHR = "1"; + + private static ExcludedRegions regions(final int start, final int end) + { + Map> map = new HashMap<>(); + map.put(CHR, new ArrayList<>(List.of(new ChrBaseRegion(CHR, start, end)))); + return new ExcludedRegions(map); + } + + @Test + public void testExcludes() + { + // span inside the region + assertTrue(regions(1000, 2000).excludes(CHR, 1500, 1600)); + + // span outside the region + assertFalse(regions(1000, 2000).excludes(CHR, 2500, 2600)); + + // different chromosome + assertFalse(regions(1000, 2000).excludes("2", 1500, 1600)); + + // end just touches the region start -- inclusive overlap must fire + assertTrue(regions(2000, 3000).excludes(CHR, 1900, 2000)); + + // start just touches the region end -- inclusive overlap must fire + assertTrue(regions(1000, 2000).excludes(CHR, 2000, 2100)); + } +} diff --git a/tars/src/test/java/com/hartwig/hmftools/tars/liftback/ExonRegionIndexTest.java b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/ExonRegionIndexTest.java new file mode 100644 index 00000000000..21d73c180ca --- /dev/null +++ b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/ExonRegionIndexTest.java @@ -0,0 +1,89 @@ +package com.hartwig.hmftools.tars.liftback; + +import static com.hartwig.hmftools.common.genome.refgenome.RefGenomeVersion.V38; +import static com.hartwig.hmftools.common.test.GeneTestUtils.addGeneData; +import static com.hartwig.hmftools.common.test.GeneTestUtils.addTransExonData; +import static com.hartwig.hmftools.common.test.GeneTestUtils.createEnsemblGeneData; +import static com.hartwig.hmftools.common.test.GeneTestUtils.createGeneDataCache; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.List; + +import com.hartwig.hmftools.common.ensemblcache.EnsemblDataCache; +import com.hartwig.hmftools.common.gene.ExonData; +import com.hartwig.hmftools.common.gene.TranscriptData; + +import org.junit.Test; + +// contains() uses binary search on merged ranges; earlier scan-back was O(N) and tripped on overlap edge cases. +public class ExonRegionIndexTest +{ + // builds an index from a single gene on ensembl chromosome "1" (keyed as chr1 under V38) whose one + // transcript carries the given exon spans. fromCache merges the spans, so overlapping/abutting spans coalesce. + private static ExonRegionIndex indexFor(final String chromosome, final List exonSpans) + { + EnsemblDataCache cache = createGeneDataCache(); + addGeneData(cache, chromosome, List.of(createEnsemblGeneData("ENSG_TEST", "TESTG", chromosome, 1, 1, 100_000))); + + TranscriptData transcript = new TranscriptData( + 1, "ENST_TEST", "ENSG_TEST", true, (byte) 1, 1, 100_000, null, null, "protein_coding", ""); + List exons = new ArrayList<>(); + int rank = 1; + for(final int[] span : exonSpans) + { + exons.add(new ExonData(1, span[0], span[1], rank++, -1, -1)); + } + transcript.setExons(exons); + addTransExonData(cache, "ENSG_TEST", List.of(transcript)); + + return ExonRegionIndex.fromCache(cache, V38); + } + + @Test + public void testContains() + { + ExonRegionIndex idx = indexFor("1", List.of(new int[] { 100, 200 }, new int[] { 300, 400 })); + + // hits include interval boundaries + assertTrue(idx.contains("chr1", 100)); + assertTrue(idx.contains("chr1", 150)); + assertTrue(idx.contains("chr1", 200)); + assertTrue(idx.contains("chr1", 300)); + assertTrue(idx.contains("chr1", 400)); + + // intronic / intergenic positions miss + assertFalse(idx.contains("chr1", 50)); + assertFalse(idx.contains("chr1", 250)); + assertFalse(idx.contains("chr1", 500)); + } + + @Test + public void testContainsAcrossOverlappingAndAbuttingIntervals() + { + ExonRegionIndex idx = indexFor("1", List.of( + new int[] { 100, 200 }, + new int[] { 150, 300 }, + new int[] { 301, 400 }, + new int[] { 600, 700 })); + + assertTrue(idx.contains("chr1", 250)); + assertTrue(idx.contains("chr1", 301)); + assertTrue(idx.contains("chr1", 400)); + assertFalse(idx.contains("chr1", 401)); + assertFalse(idx.contains("chr1", 599)); + assertTrue(idx.contains("chr1", 700)); + } + + @Test + public void testChromosomeKeyedInVersionedForm() + { + // ensembl stores bare "1"; the index keys it in the run's ref-genome form (V38 -> chr1) so lookups by + // lifted genomic contig match directly. The bare form does not match. + ExonRegionIndex idx = indexFor("1", List.of(new int[] { 100, 200 })); + assertTrue(idx.contains("chr1", 150)); + assertFalse(idx.contains("1", 150)); + } +} diff --git a/tars/src/test/java/com/hartwig/hmftools/tars/liftback/LiftBackDiscriminatorTest.java b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/LiftBackDiscriminatorTest.java new file mode 100644 index 00000000000..b26621b2cd0 --- /dev/null +++ b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/LiftBackDiscriminatorTest.java @@ -0,0 +1,452 @@ +package com.hartwig.hmftools.tars.liftback; + +import static com.hartwig.hmftools.tars.liftback.DecidingFeature.AMBIGUOUS; +import static com.hartwig.hmftools.tars.liftback.DecidingFeature.CONCORDANT; +import static com.hartwig.hmftools.tars.liftback.DecidingFeature.JUNCTION; +import static com.hartwig.hmftools.tars.liftback.DecidingFeature.JUNCTION_OVER_CONTIGUOUS; +import static com.hartwig.hmftools.tars.liftback.DecidingFeature.MULTIMAPPER; +import static com.hartwig.hmftools.tars.liftback.DecidingFeature.REF_READS_THROUGH; +import static com.hartwig.hmftools.tars.liftback.DecidingFeature.SOLE_REF; +import static com.hartwig.hmftools.tars.liftback.DecidingFeature.SOLE_TX; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.Test; + +// Direct unit coverage of the ref-vs-tx discriminator: categorize() resolves a record's lifted alignment set +// to an Outcome + DecidingFeature, and apply() executes the swap/drop that feature implies. These are the +// central manipulations that decide whether a tx-contig (spliced) placement wins over bwa's ref placement. +// Each test feeds a hand-built alignment set so the decision and its consequence are asserted in isolation, +// rather than through the full LiftBackResolver.resolve path. +public class LiftBackDiscriminatorTest +{ + private static final String CHR1 = "chr1"; + private static final String CHR2 = "chr2"; + + // anchors comfortably above TarsConstants.MIN_JUNCTION_ANCHOR so the N counts as a real junction. + private static final String TX_JUNCTION_CIGAR = "50M100N50M"; + private static final String FULL_MATCH_CIGAR = "100M"; + private static final String SOFTCLIP_CIGAR = "50M51S"; + + private static LiftedAlignment tx( + final String chrom, final int pos, final String cigar, final boolean softClipAtBoundary, final int numMismatches) + { + return new LiftedAlignment( + LiftedAlignment.AlignmentSource.XA_INPUT, "ctg_tx", pos, cigar, + chrom, pos, cigar, 0, numMismatches, + "ENST001", "ENSG001", "GENE", softClipAtBoundary, true); + } + + private static LiftedAlignment ref(final String chrom, final int pos, final String cigar) + { + return new LiftedAlignment( + LiftedAlignment.AlignmentSource.SELF, chrom, pos, cigar, + chrom, pos, cigar, 0, 0, + null, null, null, false, true); + } + + private static List set(final LiftedAlignment... alignments) + { + List list = new ArrayList<>(); + for(final LiftedAlignment alignment : alignments) + { + list.add(alignment); + } + return list; + } + + private static DecidingFeature featureOf(final List alignments) + { + return LiftBackDiscriminator.categorize(alignments).Feature; + } + + // The full ref-vs-tx decision matrix: one assert per discriminator branch. Each assert carries a message + // naming the branch so a failure pinpoints which decision misclassified. + @Test + public void testCategorize() + { + // ---- one locus, single source ---- + + // only a ref alignment present -> REF, SOLE_REF. + assertEquals("single ref alignment", SOLE_REF, + featureOf(set(ref(CHR1, 100, FULL_MATCH_CIGAR)))); + + // only a tx (spliced) alignment present -> TX, SOLE_TX. + assertEquals("single tx alignment", SOLE_TX, + featureOf(set(tx(CHR1, 100, TX_JUNCTION_CIGAR, false, 0)))); + + // ---- one locus, ref and tx both present (the discriminator) ---- + + // identical cigar, no N: the two views agree, no contest -> CONCORDANT. + assertEquals("ref and tx agree on the same gapless cigar", CONCORDANT, + featureOf(set( + ref(CHR1, 100, FULL_MATCH_CIGAR), + tx(CHR1, 100, FULL_MATCH_CIGAR, false, 0)))); + + // tx spliced across a real junction, ref ran off the exon edge into a softclip: tx wins (JUNCTION). + assertEquals("tx junction beats a ref softclip", JUNCTION, + featureOf(set( + ref(CHR1, 100, SOFTCLIP_CIGAR), + tx(CHR1, 100, TX_JUNCTION_CIGAR, false, 0)))); + + // ref matched cleanly through the supposed intron (no softclip): read is unspliced, ref wins. + // REF_READS_THROUGH merges the old TX_JUNCTION_REF_MATCH / TX_SOFTCLIP_REF_MATCH; the underlying + // flags keep the two scenarios distinguishable, so assert them too. + LiftBackDiscriminator.Features refMatch = LiftBackDiscriminator.categorize(set( + ref(CHR1, 100, FULL_MATCH_CIGAR), + tx(CHR1, 100, TX_JUNCTION_CIGAR, false, 0))); + assertEquals("clean ref match through the intron beats the tx junction", REF_READS_THROUGH, refMatch.Feature); + assertTrue(refMatch.TxHasNCigar); + assertTrue(refMatch.RefFullMatch); + assertFalse(refMatch.RefSoftClipped); + + // tx softclipped at the exon boundary (no N junction), ref full match: ref wins (intron retention). + LiftBackDiscriminator.Features txSoftclip = LiftBackDiscriminator.categorize(set( + ref(CHR1, 100, FULL_MATCH_CIGAR), + tx(CHR1, 100, "50M50S", true, 0))); + assertEquals("ref full match beats a tx that only softclips at the boundary", REF_READS_THROUGH, txSoftclip.Feature); + assertTrue(txSoftclip.TxSoftClipAtBoundary); + assertFalse(txSoftclip.TxHasNCigar); + + // tx contiguous (no junction, no boundary clip), ref softclipped: no rule applies -> AMBIGUOUS. + assertEquals("no rule fires when tx is contiguous and ref is softclipped", AMBIGUOUS, + featureOf(set( + ref(CHR1, 100, SOFTCLIP_CIGAR), + tx(CHR1, 100, FULL_MATCH_CIGAR, false, 0)))); + + // ---- two or more loci ---- + + // multiple loci, all ref, zero tx alts -> SOLE_REF. + assertEquals("multi-locus, ref only", SOLE_REF, + featureOf(set( + ref(CHR1, 100, FULL_MATCH_CIGAR), + ref(CHR2, 200, FULL_MATCH_CIGAR)))); + + // multiple loci, all tx, zero ref alts -> SOLE_TX. + assertEquals("multi-locus, tx only", SOLE_TX, + featureOf(set( + tx(CHR1, 100, TX_JUNCTION_CIGAR, false, 0), + tx(CHR2, 200, TX_JUNCTION_CIGAR, false, 0)))); + + // distinct loci, ref and tx; tx has an annotated junction, ref alt is intronless (intronless ref alt): tx faithful. + assertEquals("multi-locus with a tx junction against an intronless ref alt", JUNCTION_OVER_CONTIGUOUS, + featureOf(set( + tx(CHR1, 100, TX_JUNCTION_CIGAR, false, 0), + ref(CHR2, 200, FULL_MATCH_CIGAR)))); + + // distinct loci, neither side has a junction: genuine multi-mapper. + assertEquals("multi-locus with no junction on either side", MULTIMAPPER, + featureOf(set( + tx(CHR1, 100, FULL_MATCH_CIGAR, false, 0), + ref(CHR2, 200, FULL_MATCH_CIGAR)))); + } + + @Test + public void testCategorizeSkipsGateDroppedAlt() + { + // an XA alt the overhang gate collapsed to a contiguous alignment is marked Dropped before the + // discriminator runs; categorize must ignore it. A ref self plus a dropped tx alt at another locus + // resolves to SOLE_REF (one counted locus), not a multi-locus tx-vs-ref contest. + LiftedAlignment self = ref(CHR1, 100, FULL_MATCH_CIGAR); + LiftedAlignment droppedTx = tx(CHR2, 200, TX_JUNCTION_CIGAR, false, 0); + droppedTx.Dropped = true; + + assertEquals(SOLE_REF, featureOf(set(self, droppedTx))); + } + + // ---- apply(): tx-favouring swap (promoteTxOverRef) ---- + + @Test + public void testTxJunctionRefSoftclipSwapsRefSelfToTx() + { + // bwa picked the ref placement; tx wins, so the primary swaps to the tx alt. self stays as an alt + // (preserves its locus) but loses IsPrimaryChoice; any other ref alt is dropped. + LiftedAlignment self = ref(CHR1, 100, SOFTCLIP_CIGAR); + self.IsPrimaryChoice = true; + LiftedAlignment txWinner = tx(CHR1, 100, TX_JUNCTION_CIGAR, false, 0); + LiftedAlignment otherRef = ref(CHR2, 200, FULL_MATCH_CIGAR); + + LiftBackDiscriminator.ApplyResult outcome = + LiftBackDiscriminator.apply(set(self, txWinner, otherRef), JUNCTION, self); + + assertSame(txWinner, outcome.effectivePrimary()); + assertEquals("swapped_ref_to_tx", outcome.note()); + assertTrue(txWinner.IsPrimaryChoice); + assertFalse(self.IsPrimaryChoice); + assertFalse(self.Dropped); + assertTrue(otherRef.Dropped); + } + + @Test + public void testTxJunctionRefSoftclipTxSelfDropsRef() + { + // self is already the tx placement; keep it and drop the ref alt. + LiftedAlignment self = tx(CHR1, 100, TX_JUNCTION_CIGAR, false, 0); + self.IsPrimaryChoice = true; + LiftedAlignment refAlt = ref(CHR1, 100, SOFTCLIP_CIGAR); + + LiftBackDiscriminator.ApplyResult outcome = + LiftBackDiscriminator.apply(set(self, refAlt), JUNCTION, self); + + assertSame(self, outcome.effectivePrimary()); + assertEquals("", outcome.note()); + assertTrue(refAlt.Dropped); + } + + @Test + public void testTxSwapPrefersFewestMismatchJunctionAlt() + { + // two tx junction alts at the locus: the winner is the N-junction alt with the fewest mismatches. + LiftedAlignment self = ref(CHR1, 100, SOFTCLIP_CIGAR); + self.IsPrimaryChoice = true; + LiftedAlignment txMany = tx(CHR1, 100, "50M100N50M", false, 5); + LiftedAlignment txFew = tx(CHR1, 100, "60M100N40M", false, 2); + + LiftBackDiscriminator.ApplyResult outcome = + LiftBackDiscriminator.apply(set(self, txMany, txFew), JUNCTION, self); + + assertSame(txFew, outcome.effectivePrimary()); + } + + // ---- apply(): ref-favouring swap (promoteRefOverTx) ---- + + @Test + public void testTxJunctionRefMatchRefSelfDropsTx() + { + // ref matched through the intron; self is ref, so keep it and drop the tx alt. + LiftedAlignment self = ref(CHR1, 100, FULL_MATCH_CIGAR); + self.IsPrimaryChoice = true; + LiftedAlignment txAlt = tx(CHR1, 100, TX_JUNCTION_CIGAR, false, 0); + + LiftBackDiscriminator.ApplyResult outcome = + LiftBackDiscriminator.apply(set(self, txAlt), REF_READS_THROUGH, self); + + assertSame(self, outcome.effectivePrimary()); + assertEquals("", outcome.note()); + assertTrue(txAlt.Dropped); + } + + @Test + public void testTxJunctionRefMatchTxSelfSwapsToRef() + { + // self is the tx placement but ref wins: swap the primary to the ref alt. + LiftedAlignment self = tx(CHR1, 100, TX_JUNCTION_CIGAR, false, 0); + self.IsPrimaryChoice = true; + LiftedAlignment refWinner = ref(CHR1, 100, FULL_MATCH_CIGAR); + + LiftBackDiscriminator.ApplyResult outcome = + LiftBackDiscriminator.apply(set(self, refWinner), REF_READS_THROUGH, self); + + assertSame(refWinner, outcome.effectivePrimary()); + assertEquals("swapped_tx_to_ref", outcome.note()); + assertTrue(refWinner.IsPrimaryChoice); + assertFalse(self.IsPrimaryChoice); + } + + // ---- apply(): tx-softclip-ref-match (also REF_READS_THROUGH) ---- + + @Test + public void testTxSoftclipRefMatchRefSelfDropsTx() + { + LiftedAlignment self = ref(CHR1, 100, FULL_MATCH_CIGAR); + self.IsPrimaryChoice = true; + LiftedAlignment txAlt = tx(CHR1, 100, "50M50S", true, 0); + + LiftBackDiscriminator.ApplyResult outcome = + LiftBackDiscriminator.apply(set(self, txAlt), REF_READS_THROUGH, self); + + assertSame(self, outcome.effectivePrimary()); + assertEquals("", outcome.note()); + assertTrue(txAlt.Dropped); + } + + @Test + public void testTxSoftclipRefMatchTxSelfSwapsToRef() + { + // self is the softclipped tx placement; a clean ref full-match sits at the same locus (intron + // retention). Ref wins: swap primary to the ref alt, demote self. + LiftedAlignment self = tx(CHR1, 100, "50M50S", true, 0); + self.IsPrimaryChoice = true; + LiftedAlignment refAlt = ref(CHR1, 100, FULL_MATCH_CIGAR); + + LiftBackDiscriminator.ApplyResult outcome = + LiftBackDiscriminator.apply(set(self, refAlt), REF_READS_THROUGH, self); + + assertSame(refAlt, outcome.effectivePrimary()); + assertEquals("swapped_tx_to_ref", outcome.note()); + assertFalse(refAlt.Dropped); + } + + // ---- apply(): features with no swap ---- + + @Test + public void testNonSwappingFeaturesLeaveSelfAndDropNothing() + { + for(final DecidingFeature feature : new DecidingFeature[] { + SOLE_REF, SOLE_TX, CONCORDANT, AMBIGUOUS, MULTIMAPPER }) + { + LiftedAlignment self = ref(CHR1, 100, FULL_MATCH_CIGAR); + self.IsPrimaryChoice = true; + LiftedAlignment txAlt = tx(CHR1, 100, TX_JUNCTION_CIGAR, false, 0); + + LiftBackDiscriminator.ApplyResult outcome = + LiftBackDiscriminator.apply(set(self, txAlt), feature, self); + + assertSame("feature " + feature, self, outcome.effectivePrimary()); + assertEquals("feature " + feature, "", outcome.note()); + assertFalse("feature " + feature, self.Dropped); + assertFalse("feature " + feature, txAlt.Dropped); + } + } + + // ---- apply(): random placement when bwa gave no priority (MAPQ 0) ---- + + // AMBIGUOUS single locus (tx contiguous, ref softclipped - no shape rule fires). + private static List ambiguousSet() + { + LiftedAlignment self = ref(CHR1, 100, "50M51S"); + self.IsPrimaryChoice = true; + LiftedAlignment txAlt = tx(CHR1, 100, "100M", false, 0); + return set(self, txAlt); + } + + // MULTIMAPPER: a ref locus and a contiguous tx locus, no junction to break the tie. + private static List multimapperSet() + { + LiftedAlignment self = ref(CHR1, 100, "151M"); + self.IsPrimaryChoice = true; + LiftedAlignment altB = tx(CHR2, 200, "151M", false, 0); + return set(self, altB); + } + + @Test + public void testAmbiguousKeepsBwaOrderWhenBwaHasPriority() + { + List alignments = ambiguousSet(); + LiftedAlignment self = alignments.get(0); + LiftBackDiscriminator.ApplyResult outcome = LiftBackDiscriminator.apply(alignments, AMBIGUOUS, self, 0, true); + assertSame("bwa ranked the placements, so its primary is untouched", self, outcome.effectivePrimary()); + assertEquals("", outcome.note()); + } + + @Test + public void testAmbiguousRandomEvenSeedPicksTxAndKeepsLoserInXa() + { + List alignments = ambiguousSet(); + LiftedAlignment self = alignments.get(0); // ref, 50M51S + LiftedAlignment txAlt = alignments.get(1); // tx, 100M + LiftBackDiscriminator.ApplyResult outcome = LiftBackDiscriminator.apply(alignments, AMBIGUOUS, self, 0, false); + assertSame("even seed -> tx wins", txAlt, outcome.effectivePrimary()); + assertEquals("random_tx", outcome.note()); + assertTrue(txAlt.IsPrimaryChoice); + assertFalse(self.IsPrimaryChoice); + assertFalse("ambiguous loser is a different cigar, kept in XA not dropped", self.Dropped); + assertFalse(txAlt.Dropped); + } + + @Test + public void testAmbiguousRandomOddSeedPicksRefDeterministically() + { + LiftBackDiscriminator.ApplyResult first = + LiftBackDiscriminator.apply(ambiguousSet(), AMBIGUOUS, ambiguousSet().get(0), 1, false); + assertEquals("odd seed -> ref wins", "random_ref", first.note()); + + List repeat = ambiguousSet(); + LiftBackDiscriminator.ApplyResult second = LiftBackDiscriminator.apply(repeat, AMBIGUOUS, repeat.get(0), 1, false); + assertEquals("same seed reproduces the pick", "random_ref", second.note()); + assertSame(repeat.get(0), second.effectivePrimary()); + } + + @Test + public void testMultimapperKeepsBwaOrderWhenBwaHasPriority() + { + List alignments = multimapperSet(); + LiftedAlignment self = alignments.get(0); + LiftBackDiscriminator.ApplyResult outcome = LiftBackDiscriminator.apply(alignments, MULTIMAPPER, self, 1, true); + assertSame(self, outcome.effectivePrimary()); + assertEquals("", outcome.note()); + } + + @Test + public void testMultimapperRandomPicksAltLocusAndKeepsAll() + { + List alignments = multimapperSet(); + LiftedAlignment self = alignments.get(0); // locus chr1:100 + LiftedAlignment altB = alignments.get(1); // locus chr2:200 + LiftBackDiscriminator.ApplyResult outcome = LiftBackDiscriminator.apply(alignments, MULTIMAPPER, self, 1, false); + assertSame("seed 1 -> second locus", altB, outcome.effectivePrimary()); + assertEquals("random_locus", outcome.note()); + assertTrue(altB.IsPrimaryChoice); + assertFalse(self.IsPrimaryChoice); + assertFalse("multimapper placements all ride in XA, none dropped", self.Dropped); + assertFalse(altB.Dropped); + } + + // ---- apply(): recomputed genomic score breaks ties before the RNG ---- + + @Test + public void testAmbiguousScoreOverridesSeed() + { + // even seed would pick tx; a higher ref genomic score must win instead (not a true tie). + List alignments = ambiguousSet(); + LiftedAlignment self = alignments.get(0); // ref + LiftedAlignment txAlt = alignments.get(1); // tx + self.GenomicScore = 90; + txAlt.GenomicScore = 50; + LiftBackDiscriminator.ApplyResult outcome = LiftBackDiscriminator.apply(alignments, AMBIGUOUS, self, 0, false); + assertSame(self, outcome.effectivePrimary()); + assertEquals("score_ref", outcome.note()); + + // and the other direction: odd seed would pick ref, higher tx score wins. + List other = ambiguousSet(); + other.get(0).GenomicScore = 40; + other.get(1).GenomicScore = 88; + LiftBackDiscriminator.ApplyResult txWins = LiftBackDiscriminator.apply(other, AMBIGUOUS, other.get(0), 1, false); + assertSame(other.get(1), txWins.effectivePrimary()); + assertEquals("score_tx", txWins.note()); + } + + @Test + public void testAmbiguousEqualScoreFallsBackToSeed() + { + List alignments = ambiguousSet(); + alignments.get(0).GenomicScore = 70; + alignments.get(1).GenomicScore = 70; + LiftBackDiscriminator.ApplyResult outcome = LiftBackDiscriminator.apply(alignments, AMBIGUOUS, alignments.get(0), 0, false); + assertEquals("equal score -> seeded coin, even seed picks tx", "random_tx", outcome.note()); + assertSame(alignments.get(1), outcome.effectivePrimary()); + } + + @Test + public void testMultimapperScoreOverridesSeed() + { + // seed 0 would keep self's locus; the higher-scoring alt locus must win instead. + List alignments = multimapperSet(); + LiftedAlignment self = alignments.get(0); // chr1:100 + LiftedAlignment altB = alignments.get(1); // chr2:200 + self.GenomicScore = 60; + altB.GenomicScore = 130; + LiftBackDiscriminator.ApplyResult outcome = LiftBackDiscriminator.apply(alignments, MULTIMAPPER, self, 0, false); + assertSame(altB, outcome.effectivePrimary()); + assertEquals("score_locus", outcome.note()); + } + + @Test + public void testMultimapperEqualLocusScoreFallsBackToSeed() + { + List alignments = multimapperSet(); + LiftedAlignment self = alignments.get(0); + LiftedAlignment altB = alignments.get(1); + self.GenomicScore = 100; + altB.GenomicScore = 100; + LiftBackDiscriminator.ApplyResult outcome = LiftBackDiscriminator.apply(alignments, MULTIMAPPER, self, 1, false); + assertSame("equal scores -> seeded pick, seed 1 -> alt locus", altB, outcome.effectivePrimary()); + assertEquals("random_locus", outcome.note()); + } +} diff --git a/tars/src/test/java/com/hartwig/hmftools/tars/liftback/LiftBackEndToEndTest.java b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/LiftBackEndToEndTest.java new file mode 100644 index 00000000000..a84ed3e5350 --- /dev/null +++ b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/LiftBackEndToEndTest.java @@ -0,0 +1,216 @@ +package com.hartwig.hmftools.tars.liftback; + +import static com.hartwig.hmftools.common.test.GeneTestUtils.CHR_1; +import static com.hartwig.hmftools.tars.liftback.TarsTestFixtures.TX_CONTIG; +import static com.hartwig.hmftools.tars.liftback.TarsTestFixtures.assertLifted; +import static com.hartwig.hmftools.tars.liftback.TarsTestFixtures.primaryRecord; +import static com.hartwig.hmftools.tars.liftback.TarsTestFixtures.secondMateRecord; +import static com.hartwig.hmftools.tars.liftback.TarsTestFixtures.supplementaryRecord; +import static com.hartwig.hmftools.tars.liftback.TarsTestFixtures.threeExonContig; +import static com.hartwig.hmftools.tars.liftback.TarsTestFixtures.TestGenome; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import com.hartwig.hmftools.common.region.ChrBaseRegion; +import com.hartwig.hmftools.tars.liftback.overhang.OverhangGate; +import com.hartwig.hmftools.tars.liftback.supplementary.AnnotatedJunctionIndex; +import com.hartwig.hmftools.tars.liftback.supplementary.SupplementaryResolver; +import com.hartwig.hmftools.tars.liftback.supplementary.RefSequenceSource; +import com.hartwig.hmftools.tars.liftback.supplementary.SupplementaryConfig; + +import org.junit.Test; + +import htsjdk.samtools.SAMRecord; + +// End-to-end tests of how TarsApplication reacts to a bwa-mem2 read aligned against a transcript contig. +// Runs the FULL per-group pipeline: resolve -> overhang-gate peel -> supplementary-resolve -> post-resolve peel + +// reclaim -> mate-patch -> NH/unmap policy. (Per-component behaviour is unit-tested in +// LiftBackResolverTest / SpliceLiftBackApplyTest / the supplementary + overhang suites.) +// +// Shares the standard three-exon contig, record builders and TestGenome with TarsTestFixtures (exons +// 100-199, 300-399, 500-549; introns 200-299, 400-499; contig length 250). REF-FREE passes (translation, +// SA/mate rewrite, NH, unmap-on-lift-failure) work with the default genome; REF-DEPENDENT passes (supplementary +// resolve ref-verify, tail-extend, collapse, canonicalize) only fire when the genome bases match the read / carry a +// splice motif -- set those explicitly via TestGenome.set() in the scenario. +public class LiftBackEndToEndTest +{ + // genomic introns implied by the exon layout, registered as annotated so supplementary resolve / tail-extend recognise them. + private static Set annotatedIntrons() + { + return Set.of(new ChrBaseRegion(CHR_1, 200, 299), new ChrBaseRegion(CHR_1, 400, 499)); + } + + // chr1 of all 'A' with canonical GT..AG seeded at the two intron boundaries, so the canonicalization / + // supplementary-resolve ref-verify passes have a motif to land on. Scenarios that need matching exon bases add them via set(). + private static TestGenome scenarioGenome(final int chr1Length) + { + return new TestGenome().with(CHR_1, chr1Length, 'A') + .set(CHR_1, 200, "GT").set(CHR_1, 298, "AG") + .set(CHR_1, 400, "GT").set(CHR_1, 498, "AG"); + } + + // set read bases (needed by the ref-dependent passes: tail-extend / collapse / canonicalize compare + // the read's bases against the genome). Length should match the CIGAR's read-consuming length. + private static SAMRecord withBases(final SAMRecord record, final String bases) + { + record.setReadBases(bases.getBytes()); + return record; + } + + // ---- full-pipeline runner (mirrors LiftBackWorker's engine wiring) ---- + private static List runLiftBack(final TestGenome genome, final List reads) + { + RefSequenceSource ref = genome.asRefSource(); + AnnotatedJunctionIndex junctionIndex = new AnnotatedJunctionIndex(annotatedIntrons()); + + LiftBackResolver resolver = new LiftBackResolver(List.of(threeExonContig())); + SupplementaryResolver supplementary = + new SupplementaryResolver(junctionIndex, ref, SupplementaryConfig.enabledDefaults()); + OverhangGate overhangGate = new OverhangGate(ref); + + LiftBackStats stats = new LiftBackStats(); + LiftBackGroupProcessor processor = new LiftBackGroupProcessor( + resolver, supplementary, overhangGate, ref, null, stats); + + List emitted = new ArrayList<>(); + processor.processNameGroup(reads, new LiftedMateInfoCache(), (record, result) -> emitted.add(record)); + return emitted; + } + + // ============================ scenarios ============================ + + @Test + public void exonSpanningReadLiftsToJunctionCigar() + { + // R2 starts at tx 197 so its leading exon2 anchor is 4M (above the trimMicroAnchors threshold of 3). + SAMRecord r1 = primaryRecord("read1", TX_CONTIG, 51, "100M"); + SAMRecord r2 = secondMateRecord("read1", TX_CONTIG, 197, "50M"); + + runLiftBack(scenarioGenome(2000), List.of(r1, r2)); + + // r1 spans exon1->exon2: 50M of exon1 + 100N intron1 + 50M exon2; mate fields point at the lifted R2. + assertLifted(r1, CHR_1, 150, "50M100N50M"); + assertEquals(CHR_1, r1.getMateReferenceName()); + assertEquals(396, r1.getMateAlignmentStart()); + + assertEquals(CHR_1, r2.getReferenceName()); + assertEquals(396, r2.getAlignmentStart()); + assertTrue(r2.getCigarString().contains("N")); + assertEquals(150, r2.getMateAlignmentStart()); + } + + @Test + public void unliftableReadIsMarkedUnmapped() + { + // starts past the contig end (tx 251 with contig len 250) -> translation fails -> emitted unmapped. + SAMRecord r1 = primaryRecord("read2", TX_CONTIG, 251, "50M"); + SAMRecord r2 = secondMateRecord("read2", CHR_1, 600, "50M"); + + runLiftBack(scenarioGenome(2000), List.of(r1, r2)); + + assertTrue(r1.getReadUnmappedFlag()); + assertEquals(SAMRecord.NO_ALIGNMENT_CIGAR, r1.getCigarString()); + assertNull(r1.getStringAttribute("XA")); + + // the surviving mate loses proper-pair and is flagged mate-unmapped. + assertEquals(CHR_1, r2.getReferenceName()); + assertFalse(r2.getProperPairFlag()); + assertTrue(r2.getMateUnmappedFlag()); + } + + @Test + public void supplementarySaTagRewrittenToGenomicCoords() + { + // a split read: primary on exon1, supplementary on exon2, each pointing at the other via SA on the tx contig. + SAMRecord r1 = primaryRecord("read4", TX_CONTIG, 51, "100M"); + r1.setAttribute("SA", TX_CONTIG + ",197,+,50M,60,0;"); + SAMRecord r1Supp = supplementaryRecord("read4", TX_CONTIG, 197, "50M"); + r1Supp.setAttribute("SA", TX_CONTIG + ",51,+,100M,60,0;"); + SAMRecord r2 = secondMateRecord("read4", CHR_1, 600, "50M"); + + runLiftBack(scenarioGenome(2000), List.of(r1, r1Supp, r2)); + + // SA must be lifted to chr1 and carry no _tx contig name. + String r1Sa = r1.getStringAttribute("SA"); + assertTrue("SA should be lifted to chr1: " + r1Sa, r1Sa != null && r1Sa.startsWith(CHR_1 + ",")); + assertTrue("SA should not reference a _tx contig: " + r1Sa, !r1Sa.contains(TX_CONTIG)); + + // supplementary lifts to its exon2 genomic locus, and both records' mate fields point at R2. + assertEquals(CHR_1, r1Supp.getReferenceName()); + assertEquals(396, r1Supp.getAlignmentStart()); + assertEquals(CHR_1, r1.getMateReferenceName()); + assertEquals(600, r1.getMateAlignmentStart()); + assertEquals(CHR_1, r1Supp.getMateReferenceName()); + assertEquals(600, r1Supp.getMateAlignmentStart()); + } + + @Test + public void genomicTerminalSoftclipNotReclaimed() + { + // A GENOMIC (non-tx) read with a trailing soft-clip whose clipped bases continue contiguously in the + // genome. The standalone reclaim is tx-match-only, so a genomic over-clip is left as bwa placed it: the + // 10S is NOT walked into 60M. Genome is all 'A', so the clip bases would match - the scope guard is what + // holds it back. + TestGenome genome = scenarioGenome(2000); + SAMRecord r1 = withBases(primaryRecord("read6", CHR_1, 1000, "50M10S"), "A".repeat(60)); + SAMRecord r2 = withBases(secondMateRecord("read6", CHR_1, 1500, "50M"), "A".repeat(50)); + + runLiftBack(genome, List.of(r1, r2)); + + assertEquals("50M10S", r1.getCigarString()); + } + + @Test + public void txMatchTerminalSoftclipReclaimed() + { + // A tx-contig read (exon1, lifts to chr1:100) with a trailing soft-clip whose clipped bases continue + // contiguously in the genome. Because the chosen primary is a tx-match, the post-resolve reclaim walks the + // 10S into the reference -> 50M. Genome is all 'A' away from the intron motifs, so all-'A' bases match. + TestGenome genome = scenarioGenome(2000); + SAMRecord r1 = withBases(primaryRecord("read6b", TX_CONTIG, 1, "40M10S"), "A".repeat(50)); + SAMRecord r2 = withBases(secondMateRecord("read6b", CHR_1, 1500, "50M"), "A".repeat(50)); + + runLiftBack(genome, List.of(r1, r2)); + + assertEquals("50M", r1.getCigarString()); + assertEquals(CHR_1, r1.getReferenceName()); + assertEquals(100, r1.getAlignmentStart()); + } + + @Test + public void splitReadResolvedAcrossAnnotatedJunction() + { + // bwa split the read into a primary (50M50S, exon1 side) and a supplementary (50S50M, exon2 side) + // flanking the annotated intron 200-299. Supplementary resolve should merge them into one 50M100N50M primary and + // drop the supp. The default genome already carries the canonical GT..AG at the intron boundaries, + // so ref-verify passes. + TestGenome genome = scenarioGenome(2000); + SAMRecord primary = withBases(primaryRecord("read7", CHR_1, 150, "50M50S"), "A".repeat(100)); + SAMRecord supp = withBases(supplementaryRecord("read7", CHR_1, 300, "50S50M"), "A".repeat(100)); + SAMRecord mate = withBases(secondMateRecord("read7", CHR_1, 800, "50M"), "A".repeat(50)); + + List emitted = runLiftBack(genome, List.of(primary, supp, mate)); + + assertEquals("50M100N50M", primary.getCigarString()); + // the merged supplementary is dropped: only primary/1 + mate/2 remain. + assertEquals(2, emitted.size()); + } + + @Test + public void nativeGenomicReadPassesThroughUnchanged() + { + SAMRecord r1 = primaryRecord("read3", CHR_1, 1000, "100M"); + SAMRecord r2 = secondMateRecord("read3", CHR_1, 1100, "50M"); + + runLiftBack(scenarioGenome(2000), List.of(r1, r2)); + + assertLifted(r1, CHR_1, 1000, "100M"); + } +} diff --git a/tars/src/test/java/com/hartwig/hmftools/tars/liftback/LiftBackGroupProcessorTest.java b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/LiftBackGroupProcessorTest.java new file mode 100644 index 00000000000..7d72866412b --- /dev/null +++ b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/LiftBackGroupProcessorTest.java @@ -0,0 +1,364 @@ +package com.hartwig.hmftools.tars.liftback; + +import static com.hartwig.hmftools.common.test.GeneTestUtils.CHR_1; +import static com.hartwig.hmftools.common.bam.SamRecordUtils.SUPPLEMENTARY_ATTRIBUTE; +import static com.hartwig.hmftools.tars.liftback.TarsTestFixtures.TX_CONTIG; +import static com.hartwig.hmftools.tars.liftback.TarsTestFixtures.bases; +import static com.hartwig.hmftools.tars.liftback.TarsTestFixtures.primaryRecord; +import static com.hartwig.hmftools.tars.liftback.TarsTestFixtures.refSource; +import static com.hartwig.hmftools.tars.liftback.TarsTestFixtures.secondMateRecord; +import static com.hartwig.hmftools.tars.liftback.TarsTestFixtures.supplementaryRecord; +import static com.hartwig.hmftools.tars.liftback.TarsTestFixtures.threeExonContig; +import static com.hartwig.hmftools.tars.liftback.TarsTestFixtures.unpairedPrimaryRecord; +import static com.hartwig.hmftools.tars.liftback.TarsTestFixtures.unpairedSupplementaryRecord; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import com.hartwig.hmftools.tars.liftback.supplementary.SupplementaryResolver; +import com.hartwig.hmftools.tars.liftback.supplementary.RefSequenceSource; +import com.hartwig.hmftools.tars.liftback.supplementary.SupplementaryConfig; + +import org.junit.Test; + +import htsjdk.samtools.SAMRecord; + +// Drives LiftBackGroupProcessor.processNameGroup with a capturing sink to assert emission/drop +// decisions. Focused on the orphaned-supplementary rule: a record that stays supplementary must carry +// an SA tag (REDUX dedup's FragmentCoords.fromRead reads the primary's coords from it), so a supp whose +// SA entries all fail to lift is dropped rather than emitted with a null SA. +public class LiftBackGroupProcessorTest +{ + private static List process(final List group, final LiftBackStats stats) + { + return process(group, stats, null, null); + } + + private static List process( + final List group, final LiftBackStats stats, final SupplementaryResolver supplementaryResolver) + { + return process(group, stats, supplementaryResolver, null); + } + + // Single harness for all LiftBackGroupProcessor scenarios: the only thing tests vary is which ctor arg is + // non-default, so a ctor change touches one call site. + private static List process( + final List group, final LiftBackStats stats, final SupplementaryResolver supplementaryResolver, + final RefSequenceSource refSource) + { + LiftBackResolver resolver = new LiftBackResolver(List.of(threeExonContig())); + LiftBackGroupProcessor processor = new LiftBackGroupProcessor( + resolver, supplementaryResolver, null, refSource, null, stats); + + List emitted = new ArrayList<>(); + processor.processNameGroup(group, new LiftedMateInfoCache(), (record, result) -> emitted.add(record)); + return emitted; + } + + private static List processExcluded( + final List group, final LiftBackStats stats, final ExcludedRegions excluded) + { + LiftBackResolver resolver = new LiftBackResolver(List.of(threeExonContig())); + LiftBackGroupProcessor processor = new LiftBackGroupProcessor( + resolver, null, null, null, excluded, stats); + + List emitted = new ArrayList<>(); + processor.processNameGroup(group, new LiftedMateInfoCache(), (record, result) -> emitted.add(record)); + return emitted; + } + + private static ExcludedRegions excludedRegion(final String chromosome, final int start, final int end) + { + java.util.Map> map = new java.util.HashMap<>(); + map.put(chromosome, new ArrayList<>(List.of(new com.hartwig.hmftools.common.region.ChrBaseRegion(chromosome, start, end)))); + return new ExcludedRegions(map); + } + + @Test + public void primaryLiftingIntoExcludedRegionIsUnmapped() + { + // tx primary (exon1) lifts to chr1:100; an excluded region covering it unmaps the read REDUX-style: + // kept in the output but flagged unmapped with no cigar, not aligned in the excluded zone. + SAMRecord primary = primaryRecord(TX_CONTIG, 1, "50M"); + List emitted = processExcluded(List.of(primary), new LiftBackStats(), excludedRegion(CHR_1, 50, 300)); + + assertEquals(1, emitted.size()); + assertTrue(emitted.get(0).getReadUnmappedFlag()); + assertEquals(SAMRecord.NO_ALIGNMENT_CIGAR, emitted.get(0).getCigarString()); + } + + @Test + public void primaryOutsideExcludedRegionIsKept() + { + SAMRecord primary = primaryRecord(TX_CONTIG, 1, "50M"); // chr1:100 + List emitted = processExcluded(List.of(primary), new LiftBackStats(), excludedRegion(CHR_1, 5000, 6000)); + + assertEquals(1, emitted.size()); + assertFalse(emitted.get(0).getReadUnmappedFlag()); + } + + @Test + public void recomputesNmAgainstGenomicRefAndDropsMd() + { + // primary on the tx contig (exon1 100-199) lifts cleanly to chr1:100 50M. The genomic ref stub + // returns all 'A'; the read has two trailing mismatches -> NM must be recomputed to 2, not carried + // from the stale tx-contig NM:0, and MD must be dropped. + SAMRecord primary = primaryRecord(TX_CONTIG, 1, "50M"); + primary.setReadBases(bases("A".repeat(48) + "CC")); // two trailing mismatches vs the all-'A' genomic stub + primary.setAttribute("NM", 0); + primary.setAttribute("MD", "50"); + + String chr1Bases = "A".repeat(600); + List emitted = process(List.of(primary), new LiftBackStats(), null, refSource(CHR_1, chr1Bases)); + + assertEquals(1, emitted.size()); + SAMRecord out = emitted.get(0); + assertEquals(CHR_1, out.getReferenceName()); + assertEquals(Integer.valueOf(2), out.getIntegerAttribute("NM")); + assertNull(out.getStringAttribute("MD")); + } + + @Test + public void emittedPrimaryTaggedWithLocusCountNh() + { + // single-locus ref-only primary -> NH = 1. + SAMRecord primary = primaryRecord(TX_CONTIG, 1, "50M"); + + List emitted = process(List.of(primary), new LiftBackStats()); + + assertEquals(1, emitted.size()); + assertEquals(Integer.valueOf(1), emitted.get(0).getIntegerAttribute("NH")); + } + + @Test + public void overCapGenomicPrimaryMapq0NoXaUnmapped() + { + // a GENOMIC primary emitted MAPQ 0 with no XA maps past the XA cap (75+ distinct genomic loci), so it + // is unmapped even though, with no XA, the resolver sees a single locus and would otherwise bump to 60. + SAMRecord primary = primaryRecord(CHR_1, 100, "50M"); + primary.setMappingQuality(0); + + List emitted = process(List.of(primary), new LiftBackStats()); + + assertEquals(1, emitted.size()); + assertTrue(emitted.get(0).getReadUnmappedFlag()); + } + + @Test + public void overCapTxPrimaryMapq0NoXaKept() + { + // a TX-CONTIG primary with MAPQ 0 + no XA hit 75+ transcript contigs of one gene, which all lift to one + // genomic locus -- the over-cap rule is REF_ONLY-gated and must NOT unmap it. It lifts and is kept. + SAMRecord primary = primaryRecord(TX_CONTIG, 1, "50M"); + primary.setMappingQuality(0); + + List emitted = process(List.of(primary), new LiftBackStats()); + + assertEquals(1, emitted.size()); + assertFalse(emitted.get(0).getReadUnmappedFlag()); + assertEquals(CHR_1, emitted.get(0).getReferenceName()); + } + + @Test + public void mapq0WithXaKeptMapped() + { + // MAPQ 0 but XA present = an ordinary few-way multimapper (within the cap); keep and lift it. + SAMRecord primary = primaryRecord(TX_CONTIG, 1, "50M"); + primary.setMappingQuality(0); + primary.setAttribute("XA", CHR_1 + ",+5000,50M,0;"); + + List emitted = process(List.of(primary), new LiftBackStats()); + + assertEquals(1, emitted.size()); + assertFalse(emitted.get(0).getReadUnmappedFlag()); + } + + @Test + public void pairedMatesPatchedAgainstPerGroupCache() + { + // both mates of one fragment in the group: /1 lifts to chr1:100, /2 (exon2) to chr1:300. The + // per-group cache must let each mate's fields point at the other's lifted coords -- this is the + // single-pass per-group correctness property that replaced the whole-sample pass-1 cache. + SAMRecord mate1 = primaryRecord(TX_CONTIG, 1, "50M"); + SAMRecord mate2 = secondMateRecord(TX_CONTIG, 101, "50M"); + + List emitted = process(List.of(mate1, mate2), new LiftBackStats()); + + assertEquals(2, emitted.size()); + SAMRecord out1 = emitted.stream().filter(SAMRecord::getFirstOfPairFlag).findFirst().orElseThrow(); + SAMRecord out2 = emitted.stream().filter(record -> !record.getFirstOfPairFlag()).findFirst().orElseThrow(); + + assertEquals(CHR_1, out1.getReferenceName()); + assertEquals(100, out1.getAlignmentStart()); + assertEquals(CHR_1, out2.getReferenceName()); + assertEquals(300, out2.getAlignmentStart()); + + assertEquals(CHR_1, out1.getMateReferenceName()); + assertEquals(300, out1.getMateAlignmentStart()); + assertEquals(CHR_1, out2.getMateReferenceName()); + assertEquals(100, out2.getMateAlignmentStart()); + } + + @Test + public void singleEndPrimaryLiftedWithoutMatePatching() + { + // Ultima single-end: an unpaired primary must lift with no mate patching and without the + // getFirstOfPairFlag() throw the per-group mate-cache refresh used to hit on unpaired input. + SAMRecord primary = unpairedPrimaryRecord(TX_CONTIG, 1, "50M"); + + List emitted = process(List.of(primary), new LiftBackStats()); + + assertEquals(1, emitted.size()); + SAMRecord out = emitted.get(0); + assertFalse(out.getReadPairedFlag()); + assertFalse(out.getReadUnmappedFlag()); + assertEquals(CHR_1, out.getReferenceName()); + assertEquals(100, out.getAlignmentStart()); + assertEquals(SAMRecord.NO_ALIGNMENT_REFERENCE_NAME, out.getMateReferenceName()); + assertEquals(0, out.getInferredInsertSize()); + } + + @Test + public void singleEndPrimaryWithSupplementaryLifted() + { + // unpaired primary + its supplementary both lift and emit; exercises the mate-cache refresh with a + // supp present in the group, which is where the unpaired getFirstOfPairFlag() call sat. + SAMRecord primary = unpairedPrimaryRecord(TX_CONTIG, 100, "50M"); + SAMRecord supp = unpairedSupplementaryRecord(TX_CONTIG, 110, "30M", TX_CONTIG + ",100,+,50M,0,0;"); + + List emitted = process(List.of(primary, supp), new LiftBackStats()); + + assertEquals(2, emitted.size()); + assertTrue(emitted.stream().noneMatch(SAMRecord::getReadPairedFlag)); + assertEquals(1, emitted.stream().filter(SAMRecord::getSupplementaryAlignmentFlag).count()); + } + + @Test + public void swappedPrimarySuppSaReferencesEmittedPrimary() + { + // chimeric read: bwa's contiguous primary (chr1:5000) is swapped to a spliced tx placement (chr1:150 + // 50M100N50M); the supplementary's bwa SA still points at chr1:5000 (bwa's pre-swap primary, never emitted). + // After the swap the supp SA must be rebuilt to reference the EMITTED primary so REDUX reads the right coords. + SAMRecord primary = unpairedPrimaryRecord(CHR_1, 5000, "100M"); + primary.setAttribute("XA", TX_CONTIG + ",+51,100M,0;"); + SAMRecord supp = unpairedSupplementaryRecord(CHR_1, 8000, "40M60H", CHR_1 + ",5000,+,100M,60,0;"); + + List emitted = process(List.of(primary, supp), new LiftBackStats()); + + assertEquals(2, emitted.size()); + SAMRecord emittedSupp = emitted.stream().filter(SAMRecord::getSupplementaryAlignmentFlag).findFirst().orElseThrow(); + String sa = emittedSupp.getStringAttribute(SUPPLEMENTARY_ATTRIBUTE); + assertNotNull(sa); + assertTrue("supp SA must reference the emitted primary chr1:150, was: " + sa, + sa.startsWith(CHR_1 + ",150,+,50M100N50M")); + } + + @Test + public void duplicateSupplementariesCollapsed() + { + // two supps lifting to the same (chrom, pos, cigar, strand) collapse to one -- bwa can emit the same + // junction across multiple tx contigs. + SAMRecord primary = primaryRecord(TX_CONTIG, 1, "50M"); + SAMRecord supp1 = supplementaryRecord(TX_CONTIG, 110, "30M", TX_CONTIG + ",1,+,50M,0,0;"); + SAMRecord supp2 = supplementaryRecord(TX_CONTIG, 110, "30M", TX_CONTIG + ",1,+,50M,0,0;"); + + List emitted = process(List.of(primary, supp1, supp2), new LiftBackStats()); + + assertEquals(2, emitted.size()); + assertEquals(1, emitted.stream().filter(SAMRecord::getSupplementaryAlignmentFlag).count()); + } + + // supplementary resolver with no annotated junctions: present (so the AS-unmap gate is active) but a no-op, + // so a primary with no resolvable supps stays un-improved. + private static SupplementaryResolver noopSupplementary() + { + return new SupplementaryResolver(Collections.emptySet(), SupplementaryConfig.defaults()); + } + + @Test + public void orphanedSupplementaryIsDropped() + { + // primary lifts cleanly; supp lifts too, but its only SA entry points at an out-of-range tx + // position that fails to lift -> rewritten SA is null -> supp is dropped, not emitted. + SAMRecord primary = primaryRecord(TX_CONTIG, 100, "50M"); + SAMRecord supp = supplementaryRecord(TX_CONTIG, 110, "30M", TX_CONTIG + ",9999,+,30M,0,0;"); + + LiftBackStats stats = new LiftBackStats(); + List emitted = process(List.of(primary, supp), stats); + + assertEquals(1, emitted.size()); + assertFalse(emitted.get(0).getSupplementaryAlignmentFlag()); + assertEquals(1, stats.orphanSuppsDropped()); + } + + @Test + public void supplementaryWithLiftableSaIsKept() + { + // same shape, but the SA entry lifts -> supp is emitted with a rewritten genomic SA. + SAMRecord primary = primaryRecord(TX_CONTIG, 100, "50M"); + SAMRecord supp = supplementaryRecord(TX_CONTIG, 110, "30M", TX_CONTIG + ",100,+,50M,0,0;"); + + LiftBackStats stats = new LiftBackStats(); + List emitted = process(List.of(primary, supp), stats); + + assertEquals(2, emitted.size()); + assertEquals(0, stats.orphanSuppsDropped()); + + SAMRecord emittedSupp = emitted.stream().filter(SAMRecord::getSupplementaryAlignmentFlag).findFirst().orElse(null); + assertNotNull(emittedSupp); + String rewrittenSa = emittedSupp.getStringAttribute(SUPPLEMENTARY_ATTRIBUTE); + assertNotNull(rewrittenSa); + assertTrue(rewrittenSa.startsWith(CHR_1 + ",")); + } + + @Test + public void lowAsPrimaryUnmappedWhenLiftbackDidNotImprove() + { + // primary lifts but bwa scored it below the -T 30 floor and supplementary resolve (no junctions) can't improve it. + SAMRecord primary = primaryRecord(TX_CONTIG, 100, "50M"); + primary.setAttribute("AS", 20); + + LiftBackStats stats = new LiftBackStats(); + List emitted = process(List.of(primary), stats, noopSupplementary()); + + assertEquals(1, emitted.size()); + assertTrue(emitted.get(0).getReadUnmappedFlag()); + assertEquals(1, stats.lowAsPrimariesUnmapped()); + } + + @Test + public void highAsPrimaryKeptMapped() + { + SAMRecord primary = primaryRecord(TX_CONTIG, 100, "50M"); + primary.setAttribute("AS", 60); + + LiftBackStats stats = new LiftBackStats(); + List emitted = process(List.of(primary), stats, noopSupplementary()); + + assertEquals(1, emitted.size()); + assertFalse(emitted.get(0).getReadUnmappedFlag()); + assertEquals(0, stats.lowAsPrimariesUnmapped()); + } + + @Test + public void lowAsPrimaryKeptWhenSupplementaryResolveDisabled() + { + // without a supplementary resolver the AS-unmap gate is inactive, so a low-AS primary is left as-is. + SAMRecord primary = primaryRecord(TX_CONTIG, 100, "50M"); + primary.setAttribute("AS", 20); + + LiftBackStats stats = new LiftBackStats(); + List emitted = process(List.of(primary), stats); + + assertEquals(1, emitted.size()); + assertFalse(emitted.get(0).getReadUnmappedFlag()); + assertEquals(0, stats.lowAsPrimariesUnmapped()); + } +} diff --git a/tars/src/test/java/com/hartwig/hmftools/tars/liftback/LiftBackResolverTest.java b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/LiftBackResolverTest.java new file mode 100644 index 00000000000..39922d3d889 --- /dev/null +++ b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/LiftBackResolverTest.java @@ -0,0 +1,723 @@ +package com.hartwig.hmftools.tars.liftback; + +import static com.hartwig.hmftools.common.genome.refgenome.RefGenomeVersion.V38; +import static com.hartwig.hmftools.common.test.GeneTestUtils.CHR_1; +import static com.hartwig.hmftools.common.test.GeneTestUtils.addGeneData; +import static com.hartwig.hmftools.common.test.GeneTestUtils.addTransExonData; +import static com.hartwig.hmftools.common.test.GeneTestUtils.createEnsemblGeneData; +import static com.hartwig.hmftools.common.test.GeneTestUtils.createGeneDataCache; +import static com.hartwig.hmftools.tars.liftback.TarsTestFixtures.TX_CONTIG; +import static com.hartwig.hmftools.tars.liftback.TarsTestFixtures.threeExonContig; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.List; + +import com.hartwig.hmftools.common.ensemblcache.EnsemblDataCache; +import com.hartwig.hmftools.common.gene.ExonData; +import com.hartwig.hmftools.common.gene.TranscriptData; +import com.hartwig.hmftools.common.region.BaseRegion; +import com.hartwig.hmftools.tars.common.ContigEntry; + +import org.junit.Test; + +import htsjdk.samtools.SAMFileHeader; +import htsjdk.samtools.SAMRecord; + +// Per-category tests for LiftBackResolver plus an end-to-end smoke through LiftBackStats. +// Uses the same three-exon contig as ContigTranslatorTest so lifted-coordinate math is consistent. +public class LiftBackResolverTest +{ + private static List contigMap() + { + return List.of(threeExonContig()); + } + + // Local unpaired builder kept on purpose: the shared fixture's primaryRecord sets paired/first-of-pair/proper-pair flags + // which these resolver tests must not carry. + private static SAMRecord newRecord(final String contig, final int pos, final String cigar) + { + SAMRecord record = new SAMRecord(new SAMFileHeader()); + record.setReadName("read"); + record.setReferenceName(contig); + record.setAlignmentStart(pos); + record.setCigarString(cigar); + record.setMappingQuality(60); + return record; + } + + private static SAMRecord newUnmappedRecord() + { + SAMRecord record = new SAMRecord(new SAMFileHeader()); + record.setReadName("read"); + record.setReadUnmappedFlag(true); + return record; + } + + @Test + public void testUnmapped() + { + LiftBackResolver resolver = new LiftBackResolver(contigMap()); + LiftBackResult result = resolver.resolve(newUnmappedRecord()); + + assertEquals(RecordState.UNMAPPED, result.recordState()); + assertTrue(result.liftedAlignments().isEmpty()); + } + + @Test + public void testRefOnlyExonic() + { + SAMRecord record = newRecord(CHR_1, 1000, "150M"); + + LiftBackResolver resolver = new LiftBackResolver(contigMap()); + LiftBackResult result = resolver.resolve(record); + + assertEquals(DecidingFeature.SOLE_REF, result.decidingFeature()); + assertEquals(CHR_1, result.finalChrom()); + assertEquals(1000, result.finalPos()); + assertEquals("150M", result.finalCigar()); + assertFalse(result.hasNCigar()); + assertEquals(1, result.numLoci()); + } + + @Test + public void testTxPrimaryUniqueWithinExon() + { + SAMRecord record = newRecord(TX_CONTIG, 1, "50M"); // contig pos 1, 50M -> exon 1 (chr1:100-149) + + LiftBackResolver resolver = new LiftBackResolver(contigMap()); + LiftBackResult result = resolver.resolve(record); + + assertEquals(DecidingFeature.SOLE_TX, result.decidingFeature()); + assertEquals(CHR_1, result.finalChrom()); + assertEquals(100, result.finalPos()); + assertEquals("50M", result.finalCigar()); + assertFalse(result.hasNCigar()); + } + + @Test + public void testTxPrimaryUniqueJunctionCrosser() + { + SAMRecord record = newRecord(TX_CONTIG, 51, "100M"); // crosses exon 1 -> exon 2 (intron 200-299) + + LiftBackResolver resolver = new LiftBackResolver(contigMap()); + LiftBackResult result = resolver.resolve(record); + + assertEquals(DecidingFeature.SOLE_TX, result.decidingFeature()); + assertEquals(CHR_1, result.finalChrom()); + assertEquals(150, result.finalPos()); + assertEquals("50M100N50M", result.finalCigar()); + assertTrue(result.hasNCigar()); + } + + @Test + public void testJunctionTxBetter() + { + // Tx junction-crosser primary; ref alt at same lifted locus with soft-clip. + SAMRecord record = newRecord(TX_CONTIG, 51, "100M"); + record.setAttribute("XA", CHR_1 + ",+150,50M50S,0;"); + + LiftBackResolver resolver = new LiftBackResolver(contigMap()); + LiftBackResult result = resolver.resolve(record); + + assertEquals(DecidingFeature.JUNCTION, result.decidingFeature()); + assertTrue(result.hasNCigar()); + assertEquals(1, result.numLoci()); + // tx wins, so ref alt is dropped from the emitted set; only tx CIGAR remains at the locus. + assertEquals(1, result.numDistinctCigarsAtPrimaryLocus()); + assertEquals(2, result.liftedAlignments().size()); + assertEquals(1, result.liftedAlignments().stream().filter(la -> !la.Dropped).count()); + } + + @Test + public void testRefTxAgree() + { + SAMRecord record = newRecord(CHR_1, 100, "50M"); // ref primary; Tx alt at contig pos 1 -> same locus+CIGAR + record.setAttribute("XA", TX_CONTIG + ",+1,50M,0;"); + + LiftBackResolver resolver = new LiftBackResolver(contigMap()); + LiftBackResult result = resolver.resolve(record); + + assertEquals(DecidingFeature.CONCORDANT, result.decidingFeature()); + assertEquals(1, result.numLoci()); + } + + @Test + public void testIntronRetRefBetter() + { + // ref chr1:170 50M (last 20bp in intron); Tx pos 71 30M20S lifts to chr1:170-199 soft-clipped at exon boundary. + SAMRecord record = newRecord(CHR_1, 170, "50M"); + record.setAttribute("XA", TX_CONTIG + ",+71,30M20S,0;"); + + LiftBackResolver resolver = new LiftBackResolver(contigMap()); + LiftBackResult result = resolver.resolve(record); + + assertEquals(DecidingFeature.REF_READS_THROUGH, result.decidingFeature()); + assertEquals(1, result.numLoci()); + } + + @Test + public void testTxSoftClipNotAtBoundaryFallsToAmbiguous() + { + // ref 50M full match; Tx 25M25S - 25M ends mid-exon (chr1:194), trailing clip NOT at exon boundary -> BOTH_AMBIGUOUS. + SAMRecord record = newRecord(CHR_1, 170, "50M"); + record.setAttribute("XA", TX_CONTIG + ",+71,25M25S,0;"); + + LiftBackResolver resolver = new LiftBackResolver(contigMap()); + LiftBackResult result = resolver.resolve(record); + + assertEquals(DecidingFeature.AMBIGUOUS, result.decidingFeature()); + assertEquals(1, result.numLoci()); + } + + @Test + public void testMultiLocusTwoLoci() + { + SAMRecord record = newRecord(TX_CONTIG, 1, "50M"); // Tx primary + ref alt on different chrom -> two loci + record.setAttribute("XA", "chr5,+5000,50M,0;"); + + LiftBackResolver resolver = new LiftBackResolver(contigMap()); + LiftBackResult result = resolver.resolve(record); + + assertEquals(DecidingFeature.MULTIMAPPER, result.decidingFeature()); + assertEquals(2, result.numLoci()); + } + + @Test + public void testDistinctLocusAltBlocksMapqRescue() + { + // Tx primary (perfect) + a lower-scoring XA alt (3 mismatches) at a distinct locus. bwa left the read + // MAPQ 0; two distinct genomic loci means it stays a multimapper - TARS does not override bwa's call with a + // weaker reconstructed score - so numLoci == 2 and MAPQ is held at 0. + SAMRecord record = newRecord(TX_CONTIG, 1, "50M"); + record.setMappingQuality(0); + record.setAttribute("XA", "chr5,+5000,50M,3;"); + + LiftBackResolver resolver = new LiftBackResolver(contigMap()); + LiftBackResult result = resolver.resolve(record); + + assertEquals(2, result.numLoci()); + assertEquals(0, result.updatedMapq()); + } + + @Test + public void testNestedSpanAltCountsAsSingleLocusAndBumpsMapq() + { + // Regression: a 5'-softclipped isoform alt begins at a downstream exon of the SAME placement. It lifts to a + // different start (chr1:300) but a span (300-349) nested inside the junction-crossing primary's (chr1:150-349), + // so it is the same genomic locus, not a second one. A MAPQ-0 read here is therefore a single-locus placement + // and must bump to 60 -- keying loci on exact start (the old behaviour) miscounted this as two loci and held it at 0. + SAMRecord record = newRecord(TX_CONTIG, 51, "100M"); // -> chr1:150 50M100N50M, genomic span 150-349 + record.setMappingQuality(0); + record.setAttribute("XA", TX_CONTIG + ",+101,50S50M,0;"); // -> chr1:300 50S50M, span 300-349 (nested) + + LiftBackResolver resolver = new LiftBackResolver(contigMap()); + LiftBackResult result = resolver.resolve(record); + + assertEquals(1, result.numLoci()); + assertEquals(60, result.updatedMapq()); + } + + @Test + public void testChainedOverlapAltNotMergedThroughPrimary() + { + // A tandem-repeat-style read: primary chr1:1000-1099; alt B chr1:1080-1179 overlaps the primary; alt C + // chr1:1160-1259 overlaps B but NOT the primary. C is a genuinely distinct locus, so numLoci must be 2 and a + // MAPQ-0 read stays 0. Locus identity is anchored on the PRIMARY's span - C must not be chained back into the + // primary via B (a naive interval-merge over all alignments would wrongly collapse to one locus and bump to 60). + SAMRecord record = newRecord(CHR_1, 1000, "100M"); + record.setMappingQuality(0); + record.setAttribute("XA", CHR_1 + ",+1080,100M,0;" + CHR_1 + ",+1160,100M,0;"); + + LiftBackResolver resolver = new LiftBackResolver(contigMap()); + LiftBackResult result = resolver.resolve(record); + + assertEquals(2, result.numLoci()); + assertEquals(0, result.updatedMapq()); + } + + @Test + public void testRefOnlyMulti() + { + SAMRecord record = newRecord(CHR_1, 1000, "50M"); // ref primary + ref alt on different chrom, no tx + record.setAttribute("XA", "chr5,+5000,50M,0;"); + + LiftBackResolver resolver = new LiftBackResolver(contigMap()); + LiftBackResult result = resolver.resolve(record); + + assertEquals(DecidingFeature.SOLE_REF, result.decidingFeature()); + assertEquals(2, result.numLoci()); + } + + @Test + public void testTxPrimaryMulti() + { + ContigEntry entryB = new ContigEntry( + "ensG_OTHER_T", 1, 100, "GO", "OTHER", "TO", "chr5", 1, + List.of(new BaseRegion(2000, 2099))); + List twoContigs = List.of(threeExonContig(), entryB); + + SAMRecord record = newRecord(TX_CONTIG, 1, "50M"); + record.setAttribute("XA", "ensG_OTHER_T,+1,50M,0;"); + + LiftBackResolver resolver = new LiftBackResolver(twoContigs); + LiftBackResult result = resolver.resolve(record); + + assertEquals(DecidingFeature.SOLE_TX, result.decidingFeature()); + assertEquals(2, result.numLoci()); + } + + @Test + public void testSupplementary() + { + SAMRecord record = newRecord(CHR_1, 1000, "60M40H"); + record.setSupplementaryAlignmentFlag(true); + + LiftBackResolver resolver = new LiftBackResolver(contigMap()); + LiftBackResult result = resolver.resolve(record); + + assertEquals(RecordState.SUPPLEMENTARY, result.recordState()); + assertEquals(LiftBackResult.RecordRole.SUPPLEMENTARY, result.role()); + } + + @Test + public void testSupplementaryOnTxContigGetsLifted() + { + SAMRecord record = newRecord(TX_CONTIG, 51, "100M"); + record.setSupplementaryAlignmentFlag(true); + + LiftBackResolver resolver = new LiftBackResolver(contigMap()); + LiftBackResult result = resolver.resolve(record); + + assertEquals(RecordState.SUPPLEMENTARY, result.recordState()); + assertEquals(LiftBackResult.RecordRole.SUPPLEMENTARY, result.role()); + assertEquals(CHR_1, result.finalChrom()); + assertEquals(150, result.finalPos()); + assertEquals("50M100N50M", result.finalCigar()); + assertTrue(result.hasNCigar()); + } + + @Test + public void testSupplementaryOnTxContigUnliftablePastEnd() + { + SAMRecord record = newRecord(TX_CONTIG, 251, "10M"); // pos 251 past altEnd(250) + record.setSupplementaryAlignmentFlag(true); + + LiftBackResolver resolver = new LiftBackResolver(contigMap()); + LiftBackResult result = resolver.resolve(record); + + assertEquals(RecordState.LIFT_FAILED, result.recordState()); + assertEquals(LiftBackResult.RecordRole.SUPPLEMENTARY, result.role()); + assertTrue(result.liftedAlignments().isEmpty()); + } + + @Test + public void testPrimaryUnliftablePastEnd() + { + SAMRecord record = newRecord(TX_CONTIG, 251, "10M"); + + LiftBackResolver resolver = new LiftBackResolver(contigMap()); + LiftBackResult result = resolver.resolve(record); + + assertEquals(RecordState.LIFT_FAILED, result.recordState()); + assertEquals(LiftBackResult.RecordRole.PRIMARY, result.role()); + assertTrue(result.notes().contains("primary_translate_failed")); + } + + @Test + public void testPrimaryTrailingOverhangClampedToSoftClip() + { + SAMRecord record = newRecord(TX_CONTIG, 200, "100M"); // 49bp past altEnd(250) -> trailing 49S + + LiftBackResolver resolver = new LiftBackResolver(contigMap()); + LiftBackResult result = resolver.resolve(record); + + assertEquals(DecidingFeature.SOLE_TX, result.decidingFeature()); + assertTrue(result.finalCigar().endsWith("49S")); + } + + @Test + public void testIntronRetRefBetterLeadingSoftClipBoundary() + { + // Tx alt contig pos 101 with leading 20S at exon-1/exon-2 boundary; ref chr1:300 full match. + SAMRecord record = newRecord(CHR_1, 300, "30M"); + record.setAttribute("XA", TX_CONTIG + ",+101,20S30M,0;"); + + LiftBackResolver resolver = new LiftBackResolver(contigMap()); + LiftBackResult result = resolver.resolve(record); + + assertEquals(DecidingFeature.REF_READS_THROUGH, result.decidingFeature()); + assertEquals(1, result.numLoci()); + } + + @Test + public void testXaDedupDropsDuplicateAlts() + { + SAMRecord record = newRecord(TX_CONTIG, 1, "50M"); // two identical XA entries -> one alt retained + record.setAttribute("XA", "chr5,+5000,50M,0;chr5,+5000,50M,0;"); + + LiftBackResolver resolver = new LiftBackResolver(contigMap()); + LiftBackResult result = resolver.resolve(record); + + assertEquals(2, result.liftedAlignments().size()); // self + one deduped alt + assertEquals(1, result.numXaAlts()); + } + + @Test + public void testXaDedupKeepsAltMatchingSelfButDropsXaDuplicate() + { + // Two XA entries lifting to the same (chr1,100,50M): one Tx, one ref. XA dedup is XA-internal only, + // so the Tx alt is kept (drives BOTH_AGREE); the duplicate ref XA collapses. + SAMRecord record = newRecord(CHR_1, 100, "50M"); + record.setAttribute("XA", TX_CONTIG + ",+1,50M,0;" + CHR_1 + ",+100,50M,0;"); + + LiftBackResolver resolver = new LiftBackResolver(contigMap()); + LiftBackResult result = resolver.resolve(record); + + assertEquals(2, result.liftedAlignments().size()); // self + Tx alt; ref XA collapsed + assertEquals(1, result.numXaAlts()); + assertEquals(DecidingFeature.CONCORDANT, result.decidingFeature()); + } + + @Test + public void testXaWithMalformedNmStillLifted() + { + SAMRecord record = newRecord(TX_CONTIG, 1, "50M"); // garbled NM field must not silently drop the alt + record.setAttribute("XA", "chr5,+5000,50M,not_a_number;"); + + LiftBackResolver resolver = new LiftBackResolver(contigMap()); + LiftBackResult result = resolver.resolve(record); + + assertEquals(2, result.liftedAlignments().size()); + assertEquals(DecidingFeature.MULTIMAPPER, result.decidingFeature()); + } + + @Test + public void testCrossLocusFavoursTxSwapsToTx() + { + // bwa picked contiguous chr5:5000 as primary; Tx XA lifts to chr1:150 50M100N50M. Expect swap to Tx, MAPQ rescued. + SAMRecord record = newRecord("chr5", 5000, "100M"); + record.setAttribute("XA", TX_CONTIG + ",+51,100M,0;"); + record.setMappingQuality(60); + + LiftBackResolver resolver = new LiftBackResolver(contigMap()); + LiftBackResult result = resolver.resolve(record); + + assertEquals(DecidingFeature.JUNCTION_OVER_CONTIGUOUS, result.decidingFeature()); + assertEquals(CHR_1, result.finalChrom()); + assertEquals(150, result.finalPos()); + assertEquals("50M100N50M", result.finalCigar()); + assertTrue(result.hasNCigar()); + assertEquals(60, result.updatedMapq()); + assertTrue(result.notes().contains("swapped_ref_to_tx")); + + LiftedAlignment originalSelf = result.liftedAlignments().stream() + .filter(la -> la.Source == LiftedAlignment.AlignmentSource.SELF) + .findFirst().orElseThrow(); + assertFalse(originalSelf.IsPrimaryChoice); + assertFalse(originalSelf.Dropped); + + LiftedAlignment winner = result.liftedAlignments().stream() + .filter(la -> la.IsPrimaryChoice) + .findFirst().orElseThrow(); + assertTrue(winner.fromTxContig()); + assertEquals("50M100N50M", winner.LiftedCigar); + } + + @Test + public void testCrossLocusBothSplicedRemainsMultiLocus() + { + // Both Tx primary and ref XA alt have spliced CIGARs at different loci - ambiguous, must NOT swap. + SAMRecord record = newRecord(TX_CONTIG, 51, "100M"); + record.setAttribute("XA", "chr5,+5000,50M100N50M,0;"); + + LiftBackResolver resolver = new LiftBackResolver(contigMap()); + LiftBackResult result = resolver.resolve(record); + + assertEquals(DecidingFeature.MULTIMAPPER, result.decidingFeature()); + LiftedAlignment self = result.liftedAlignments().stream() + .filter(la -> la.Source == LiftedAlignment.AlignmentSource.SELF) + .findFirst().orElseThrow(); + assertTrue(self.IsPrimaryChoice); + } + + @Test + public void testCrossLocusFavoursTxDropsOtherRefAlts() + { + // Tx XA wins (chr1:150 spliced); original self (chr5) kept as an informative alt; chr7 ref alt dropped. + SAMRecord record = newRecord("chr5", 5000, "100M"); + record.setAttribute("XA", TX_CONTIG + ",+51,100M,0;chr7,+7000,100M,0;"); + + LiftBackResolver resolver = new LiftBackResolver(contigMap()); + LiftBackResult result = resolver.resolve(record); + + assertEquals(DecidingFeature.JUNCTION_OVER_CONTIGUOUS, result.decidingFeature()); + + LiftedAlignment chr7Alt = result.liftedAlignments().stream() + .filter(la -> "chr7".equals(la.LiftedChrom)) + .findFirst().orElseThrow(); + assertTrue(chr7Alt.Dropped); + + LiftedAlignment chr5Self = result.liftedAlignments().stream() + .filter(la -> "chr5".equals(la.LiftedChrom)) + .findFirst().orElseThrow(); + assertFalse(chr5Self.Dropped); + assertFalse(chr5Self.IsPrimaryChoice); + } + + @Test + public void testJunctionFavoursTxSwapsWhenSelfIsRef() + { + // bwa picked ref soft-clipped (50M50S) as primary; Tx XA lifts to the same locus with 50M100N50M. Tx wins and swaps. + SAMRecord record = newRecord(CHR_1, 150, "50M50S"); + record.setAttribute("XA", TX_CONTIG + ",+51,100M,0;"); + + LiftBackResolver resolver = new LiftBackResolver(contigMap()); + LiftBackResult result = resolver.resolve(record); + + assertEquals(DecidingFeature.JUNCTION, result.decidingFeature()); + assertEquals("50M100N50M", result.finalCigar()); + assertTrue(result.hasNCigar()); + assertTrue(result.notes().contains("swapped_ref_to_tx")); + } + + @Test + public void testJunctionRefMatchKeepsRefAndDropsTx() + { + // Ref full-match 151M through the supposed intron is overwhelming evidence of an unspliced read; keep ref, drop Tx. + SAMRecord record = newRecord(CHR_1, 150, "151M"); + record.setAttribute("XA", TX_CONTIG + ",+51,100M,0;"); + + LiftBackResolver resolver = new LiftBackResolver(contigMap()); + LiftBackResult result = resolver.resolve(record); + + assertEquals(DecidingFeature.REF_READS_THROUGH, result.decidingFeature()); + assertEquals(CHR_1, result.finalChrom()); + assertEquals(150, result.finalPos()); + assertEquals("151M", result.finalCigar()); + assertFalse(result.hasNCigar()); + + LiftedAlignment self = result.liftedAlignments().stream() + .filter(la -> la.Source == LiftedAlignment.AlignmentSource.SELF) + .findFirst().orElseThrow(); + assertTrue(self.IsPrimaryChoice); + assertFalse(self.Dropped); + + LiftedAlignment txAlt = result.liftedAlignments().stream() + .filter(LiftedAlignment::fromTxContig) + .findFirst().orElseThrow(); + assertTrue(txAlt.Dropped); + assertFalse(txAlt.IsPrimaryChoice); + } + + @Test + public void testJunctionRefMatchSwapsToRefWhenSelfIsTx() + { + // Tx primary; ref XA is unspliced full-match. Discriminator swaps to ref and drops Tx alt. + SAMRecord record = newRecord(TX_CONTIG, 51, "100M"); + record.setAttribute("XA", CHR_1 + ",+150,151M,0;"); + + LiftBackResolver resolver = new LiftBackResolver(contigMap()); + LiftBackResult result = resolver.resolve(record); + + assertEquals(DecidingFeature.REF_READS_THROUGH, result.decidingFeature()); + assertEquals(CHR_1, result.finalChrom()); + assertEquals(150, result.finalPos()); + assertEquals("151M", result.finalCigar()); + assertFalse(result.hasNCigar()); + assertTrue(result.notes().contains("swapped_tx_to_ref")); + + LiftedAlignment txSelf = result.liftedAlignments().stream() + .filter(la -> la.Source == LiftedAlignment.AlignmentSource.SELF) + .findFirst().orElseThrow(); + assertFalse(txSelf.IsPrimaryChoice); + assertFalse(txSelf.Dropped); // demoted to informative XA, not dropped + + LiftedAlignment winner = result.liftedAlignments().stream() + .filter(la -> la.IsPrimaryChoice) + .findFirst().orElseThrow(); + assertFalse(winner.fromTxContig()); + } + + @Test + public void testStatsEndToEndSmoke() + { + LiftBackResolver resolver = new LiftBackResolver(contigMap()); + LiftBackStats stats = new LiftBackStats(); + + SAMRecord r1 = newRecord(CHR_1, 1000, "150M"); // REF_SINGLE + stats.record(r1, resolver.resolve(r1)); + + SAMRecord r2 = newRecord(TX_CONTIG, 51, "100M"); // TX_SINGLE, junction-crosser + stats.record(r2, resolver.resolve(r2)); + + SAMRecord r3 = newRecord(CHR_1, 100, "50M"); // BOTH_AGREE, MAPQ=0 + r3.setAttribute("XA", TX_CONTIG + ",+1,50M,0;"); + r3.setMappingQuality(0); + stats.record(r3, resolver.resolve(r3)); + + SAMRecord r4 = newUnmappedRecord(); // UNMAPPED + stats.record(r4, resolver.resolve(r4)); + + assertEquals(4, stats.total()); + assertEquals(1, stats.featureCount(DecidingFeature.SOLE_REF)); + assertEquals(1, stats.featureCount(DecidingFeature.SOLE_TX)); + assertEquals(1, stats.featureCount(DecidingFeature.CONCORDANT)); + assertEquals(1, stats.stateCount(RecordState.UNMAPPED)); + } + + // numLoci must reflect the deduped genomic-locus count, not the XA entry count (NH is derived from it). + @Test + public void testNumLociDedupesIdenticalLiftedXaEntries() + { + SAMRecord primary = newRecord(TX_CONTIG, 51, "100M"); // lifts to chr1:150 50M100N50M + // four XA entries all lifting to the same locus -> numLoci still 1 + primary.setAttribute("XA", + TX_CONTIG + ",+51,100M,0;" + + TX_CONTIG + ",+51,100M,0;" + + TX_CONTIG + ",+51,100M,0;" + + TX_CONTIG + ",+51,100M,0;"); + + LiftBackResolver resolver = new LiftBackResolver(contigMap()); + LiftBackResult result = resolver.resolve(primary); + + assertEquals(CHR_1, result.finalChrom()); + assertEquals(150, result.finalPos()); + assertEquals("50M100N50M", result.finalCigar()); + assertEquals(1, result.numLoci()); + } + + @Test + public void testNumLociCountsDistinctLiftedLoci() + { + SAMRecord primary = newRecord(CHR_1, 1000, "150M"); + primary.setAttribute("XA", + CHR_1 + ",+2000,150M,0;" + + CHR_1 + ",+3000,150M,0;"); + + LiftBackResolver resolver = new LiftBackResolver(contigMap()); + LiftBackResult result = resolver.resolve(primary); + + assertEquals(3, result.numLoci()); + } + + // Builds an ExonRegionIndex from an in-memory ensembl cache (shared by rescue-path tests). + private static ExonRegionIndex buildExonIndex(final List exons) + { + EnsemblDataCache cache = createGeneDataCache(); + addGeneData(cache, CHR_1, List.of(createEnsemblGeneData("ENSG_TEST", "TESTG", CHR_1, 1, 1, 100000))); + + TranscriptData transcript = new TranscriptData( + 1, "ENST_TEST", "ENSG_TEST", true, (byte) 1, 1, 100000, null, null, "protein_coding", ""); + List exonData = new ArrayList<>(); + int rank = 1; + for(final int[] exon : exons) + { + exonData.add(new ExonData(1, exon[0], exon[1], rank++, -1, -1)); + } + transcript.setExons(exonData); + addTransExonData(cache, "ENSG_TEST", List.of(transcript)); + + return ExonRegionIndex.fromCache(cache, V38); + } + + // Hidden tie (XS==AS) on a ref-only primary landing outside any indexed exon: no vouching evidence, so the + // unresolved hidden tie holds MAPQ at 0 (the equal-scoring alt bwa did not emit may be real). + @Test + public void testHiddenTieRefOnlyOutsideIndexedExonHoldsAtZero() throws Exception + { + SAMRecord record = newRecord(CHR_1, 1500, "150M"); + record.setMappingQuality(0); + record.setAttribute("AS", 151); + record.setAttribute("XS", 151); + + LiftBackResolver noIndex = new LiftBackResolver(contigMap()); + assertEquals(0, noIndex.resolve(record).updatedMapq()); + + ExonRegionIndex idx = buildExonIndex(List.of(new int[] { 1400, 1700 })); + LiftBackResolver withIndex = new LiftBackResolver(contigMap(), idx); + LiftBackResult result = withIndex.resolve(record); + assertEquals(0, result.updatedMapq()); + assertEquals(DecidingFeature.SOLE_REF, result.decidingFeature()); + } + + // Hidden tie with primary outside any annotated exon: rescue stays blocked. + @Test + public void testHiddenTieOutsideExonKeepsMapqZero() throws Exception + { + SAMRecord record = newRecord(CHR_1, 5000, "150M"); + record.setMappingQuality(0); + record.setAttribute("AS", 151); + record.setAttribute("XS", 151); + + ExonRegionIndex idx = buildExonIndex(List.of(new int[] { 1400, 1700 })); // exon at 1400-1700; primary at 5000 is intergenic + LiftBackResolver resolver = new LiftBackResolver(contigMap(), idx); + assertEquals(0, resolver.resolve(record).updatedMapq()); + } + + // Direct unit tests for the extracted MAPQ policy; independent of LiftBackDiscriminator / SAMRecord plumbing. + // decidePrimaryMapq positional args: (inputMapq, numLoci, hiddenTie, primaryFromTxContig, primaryInAnnotatedExon). + @Test + public void testMapqPolicy_singleLocusZeroRescues() + { + assertEquals(60, LiftBackResolver.decidePrimaryMapq(0, 1, false, false, false)); // single locus, MAPQ0, no hidden tie -> 60 + } + + @Test + public void testMapqPolicy_hiddenTieRefPrimaryNoExonHoldsAtZero() + { + assertEquals(0, LiftBackResolver.decidePrimaryMapq(0, 1, true, false, false)); // unresolved hidden tie holds at 0 + } + + @Test + public void testMapqPolicy_hiddenTieTxPrimaryRescues() + { + assertEquals(60, LiftBackResolver.decidePrimaryMapq(0, 1, true, true, false)); // tx provenance overrides hidden tie + } + + @Test + public void testMapqPolicy_hiddenTieInAnnotatedExonRescues() + { + assertEquals(60, LiftBackResolver.decidePrimaryMapq(0, 1, true, false, true)); // annotated exon overrides hidden tie + } + + @Test + public void testMapqPolicy_inputSixtyPassesAsRescued() + { + assertEquals(60, LiftBackResolver.decidePrimaryMapq(60, 1, false, false, false)); // input 60 passes through unchanged + } + + @Test + public void testMapqPolicy_gradedMapqPassesThrough() + { + assertEquals(37, LiftBackResolver.decidePrimaryMapq(37, 1, false, false, false)); // graded signal; leave alone + } + + @Test + public void testMapqPolicy_multiLocusNeverBumps() + { + assertEquals(0, LiftBackResolver.decidePrimaryMapq(0, 2, false, false, false)); // multi-locus never bumped + } + + @Test + public void testMapqPolicy_refOnlySingleLocusBumps() + { + assertEquals(60, LiftBackResolver.decidePrimaryMapq(0, 1, false, false, false)); // single-locus ref-only MAPQ0 now bumps + } + + @Test + public void testMapqPolicy_ambiguousNeverBumps() + { + // a coin-flipped tx-vs-ref call at one locus stays MAPQ 0, even though single-locus MAPQ0 would otherwise bump + assertEquals(0, LiftBackResolver.decidePrimaryMapq(0, 1, false, false, false, DecidingFeature.AMBIGUOUS)); + } + + @Test + public void testMapqPolicy_nonAmbiguousSingleLocusStillBumps() + { + assertEquals(60, LiftBackResolver.decidePrimaryMapq(0, 1, false, false, false, DecidingFeature.SOLE_REF)); + } +} diff --git a/tars/src/test/java/com/hartwig/hmftools/tars/liftback/LiftBackScenario.java b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/LiftBackScenario.java new file mode 100644 index 00000000000..184a908bd53 --- /dev/null +++ b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/LiftBackScenario.java @@ -0,0 +1,321 @@ +package com.hartwig.hmftools.tars.liftback; + +import static com.hartwig.hmftools.tars.liftback.TarsTestFixtures.primaryRecord; +import static com.hartwig.hmftools.tars.liftback.TarsTestFixtures.secondMateRecord; +import static com.hartwig.hmftools.tars.liftback.TarsTestFixtures.supplementaryRecord; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import com.hartwig.hmftools.common.region.ChrBaseRegion; +import com.hartwig.hmftools.tars.common.ContigEntry; +import com.hartwig.hmftools.tars.liftback.TarsTestFixtures.TestGenome; +import com.hartwig.hmftools.tars.liftback.overhang.OverhangGate; +import com.hartwig.hmftools.tars.liftback.supplementary.AnnotatedJunctionIndex; +import com.hartwig.hmftools.tars.liftback.supplementary.SupplementaryResolver; +import com.hartwig.hmftools.tars.liftback.supplementary.RefSequenceSource; +import com.hartwig.hmftools.tars.liftback.supplementary.SupplementaryConfig; + +import htsjdk.samtools.SAMRecord; + +// Parametric, standalone whole-pipeline test harness. Declare the transcript-contig geometry, the reference genome, +// the annotated junctions and a set of reads (primary + mate + any supplementaries, each with arbitrary tags), then +// run() drives the real LiftBackGroupProcessor.processNameGroup exactly as production does (overhang gate -> +// discriminator -> supplementary resolve -> reclaim -> mate patch -> NH / unmap policy). Assert the lifted +// placement of every emitted record. +// +// Usage: +// LiftBackScenario.create() +// .contig(TarsTestFixtures.threeExonContig()) +// .genome(new TestGenome().with(CHR_1, 5000, 'A')) +// .annotatedIntron(CHR_1, 200, 299) +// .read(primary("frag1", TX_CONTIG, 51, "100M").mapq(0).as(151).xa("chr1_tx,+8725640,100M,0;")) +// .read(mate("frag1", TX_CONTIG, 197, "50M")) +// .read(supp("frag1", TX_CONTIG, 300, "50S50M").sa("chr1_tx,51,+,100M,60,0;")) +// .run() +// .assertLifted("frag1", ReadRole.PRIMARY, CHR_1, 150, "50M100N50M") +// .assertMapq("frag1", ReadRole.PRIMARY, 60) +// .assertNoXa("frag1", ReadRole.PRIMARY) +// .assertSuppCount("frag1", 0); +public final class LiftBackScenario +{ + private final List mContigs = new ArrayList<>(); + private final Set mAnnotatedIntrons = new HashSet<>(); + private final List mReads = new ArrayList<>(); + private TestGenome mGenome; + private ExonRegionIndex mExonIndex; + + public enum ReadRole + { + PRIMARY, // first-of-pair primary + MATE, // second-of-pair primary + SUPPLEMENTARY // a supplementary (first-of-pair) + } + + public static LiftBackScenario create() + { + return new LiftBackScenario(); + } + + public LiftBackScenario contig(final ContigEntry entry) + { + mContigs.add(entry); + return this; + } + + public LiftBackScenario genome(final TestGenome genome) + { + mGenome = genome; + return this; + } + + public LiftBackScenario annotatedIntron(final String chromosome, final int start, final int end) + { + mAnnotatedIntrons.add(new ChrBaseRegion(chromosome, start, end)); + return this; + } + + // optional: drives the hidden-tie exon override in the MAPQ policy. + public LiftBackScenario exonIndex(final ExonRegionIndex exonIndex) + { + mExonIndex = exonIndex; + return this; + } + + public LiftBackScenario read(final ReadSpec spec) + { + mReads.add(spec); + return this; + } + + // ----- read factories: build a record for the given role, then chain tag setters ----- + + public static ReadSpec primary(final String readName, final String contig, final int pos, final String cigar) + { + return new ReadSpec(readName, ReadRole.PRIMARY, primaryRecord(readName, contig, pos, cigar)); + } + + public static ReadSpec mate(final String readName, final String contig, final int pos, final String cigar) + { + return new ReadSpec(readName, ReadRole.MATE, secondMateRecord(readName, contig, pos, cigar)); + } + + public static ReadSpec supp(final String readName, final String contig, final int pos, final String cigar) + { + // supplementaryRecord(readName, contig, pos, cigar) leaves the SA tag null; set it with .sa(...) when needed. + return new ReadSpec(readName, ReadRole.SUPPLEMENTARY, supplementaryRecord(readName, contig, pos, cigar)); + } + + // ----- run the full pipeline and collect every emitted record ----- + + public Result run() + { + RefSequenceSource ref = mGenome != null ? mGenome.asRefSource() : null; + AnnotatedJunctionIndex junctionIndex = new AnnotatedJunctionIndex(mAnnotatedIntrons); + + LiftBackResolver resolver = mExonIndex != null + ? new LiftBackResolver(mContigs, mExonIndex) + : new LiftBackResolver(mContigs); + SupplementaryResolver supplementary = new SupplementaryResolver(junctionIndex, ref, SupplementaryConfig.enabledDefaults()); + OverhangGate overhangGate = new OverhangGate(ref); + + LiftBackGroupProcessor processor = new LiftBackGroupProcessor( + resolver, supplementary, overhangGate, ref, null, new LiftBackStats()); + + List emitted = new ArrayList<>(); + for(final List group : groupByReadName()) + { + processor.processNameGroup(group, new LiftedMateInfoCache(), (record, result) -> emitted.add(record)); + } + return new Result(emitted); + } + + // reads sharing a name form one name-group; preserve declaration order across groups. + private List> groupByReadName() + { + List> groups = new ArrayList<>(); + List order = new ArrayList<>(); + for(final ReadSpec spec : mReads) + { + int index = order.indexOf(spec.ReadName); + if(index < 0) + { + order.add(spec.ReadName); + groups.add(new ArrayList<>()); + index = groups.size() - 1; + } + groups.get(index).add(spec.Record); + } + return groups; + } + + // A record plus its role, with fluent tag setters. SEQ defaults to the fixture bases; override with bases(...) + // when a ref-dependent pass (overhang gate / supplementary-resolve ref-verify / canonicalize) must compare read vs genome. + public static final class ReadSpec + { + final String ReadName; + final ReadRole Role; + final SAMRecord Record; + + ReadSpec(final String readName, final ReadRole role, final SAMRecord record) + { + ReadName = readName; + Role = role; + Record = record; + } + + public ReadSpec mapq(final int mapq) + { + Record.setMappingQuality(mapq); + return this; + } + + public ReadSpec bases(final String sequence) + { + Record.setReadBases(TarsTestFixtures.bases(sequence)); + return this; + } + + public ReadSpec xa(final String xa) + { + Record.setAttribute("XA", xa); + return this; + } + + public ReadSpec sa(final String sa) + { + Record.setAttribute("SA", sa); + return this; + } + + public ReadSpec as(final int alignmentScore) + { + Record.setAttribute("AS", alignmentScore); + return this; + } + + public ReadSpec nm(final int numMismatches) + { + Record.setAttribute("NM", numMismatches); + return this; + } + + public ReadSpec tag(final String name, final Object value) + { + Record.setAttribute(name, value); + return this; + } + } + + // Emitted records keyed by (readName, role), with assertion helpers over the lifted placement. + public static final class Result + { + private final List mEmitted; + + Result(final List emitted) + { + mEmitted = emitted; + } + + public List emitted() + { + return mEmitted; + } + + public SAMRecord record(final String readName, final ReadRole role) + { + for(final SAMRecord record : mEmitted) + { + if(!record.getReadName().equals(readName)) + { + continue; + } + if(matchesRole(record, role)) + { + return record; + } + } + return null; + } + + private static boolean matchesRole(final SAMRecord record, final ReadRole role) + { + return switch(role) + { + case PRIMARY -> !record.getSupplementaryAlignmentFlag() + && (!record.getReadPairedFlag() || record.getFirstOfPairFlag()); + case MATE -> !record.getSupplementaryAlignmentFlag() + && record.getReadPairedFlag() && !record.getFirstOfPairFlag(); + case SUPPLEMENTARY -> record.getSupplementaryAlignmentFlag(); + }; + } + + public Result assertLifted(final String readName, final ReadRole role, final String chrom, final int pos, final String cigar) + { + SAMRecord record = record(readName, role); + assertNotNull("no emitted record for " + readName + "/" + role, record); + assertEquals(readName + "/" + role + " chrom", chrom, record.getReferenceName()); + assertEquals(readName + "/" + role + " pos", pos, record.getAlignmentStart()); + assertEquals(readName + "/" + role + " cigar", cigar, record.getCigarString()); + return this; + } + + public Result assertMapq(final String readName, final ReadRole role, final int mapq) + { + SAMRecord record = record(readName, role); + assertNotNull("no emitted record for " + readName + "/" + role, record); + assertEquals(readName + "/" + role + " mapq", mapq, record.getMappingQuality()); + return this; + } + + public Result assertNoXa(final String readName, final ReadRole role) + { + SAMRecord record = record(readName, role); + assertNotNull("no emitted record for " + readName + "/" + role, record); + assertNull(readName + "/" + role + " XA should be absent", record.getStringAttribute("XA")); + return this; + } + + public Result assertXa(final String readName, final ReadRole role, final String xa) + { + SAMRecord record = record(readName, role); + assertNotNull("no emitted record for " + readName + "/" + role, record); + assertEquals(readName + "/" + role + " XA", xa, record.getStringAttribute("XA")); + return this; + } + + public Result assertUnmapped(final String readName, final ReadRole role) + { + SAMRecord record = record(readName, role); + assertNotNull("no emitted record for " + readName + "/" + role, record); + assertEquals(readName + "/" + role + " should be unmapped", true, record.getReadUnmappedFlag()); + return this; + } + + public Result assertSuppCount(final String readName, final int expected) + { + int count = 0; + for(final SAMRecord record : mEmitted) + { + if(record.getReadName().equals(readName) && record.getSupplementaryAlignmentFlag()) + { + ++count; + } + } + assertEquals(readName + " supplementary count", expected, count); + return this; + } + + public Result assertEmittedCount(final int expected) + { + assertEquals("total emitted records", expected, mEmitted.size()); + return this; + } + } +} diff --git a/tars/src/test/java/com/hartwig/hmftools/tars/liftback/LiftBackScenarioTest.java b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/LiftBackScenarioTest.java new file mode 100644 index 00000000000..73e6d02e101 --- /dev/null +++ b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/LiftBackScenarioTest.java @@ -0,0 +1,112 @@ +package com.hartwig.hmftools.tars.liftback; + +import static com.hartwig.hmftools.common.test.GeneTestUtils.CHR_1; +import static com.hartwig.hmftools.tars.liftback.LiftBackScenario.ReadRole.PRIMARY; +import static com.hartwig.hmftools.tars.liftback.LiftBackScenario.mate; +import static com.hartwig.hmftools.tars.liftback.LiftBackScenario.primary; +import static com.hartwig.hmftools.tars.liftback.LiftBackScenario.supp; +import static com.hartwig.hmftools.tars.liftback.TarsTestFixtures.TX_CONTIG; +import static com.hartwig.hmftools.tars.liftback.TarsTestFixtures.threeExonContig; + +import com.hartwig.hmftools.tars.liftback.TarsTestFixtures.TestGenome; + +import org.junit.Test; + +// Demonstrates the parametric LiftBackScenario harness: each test declares reads + tags + geometry and asserts the +// full pass-through result. Outcomes mirror the hand-wired LiftBackEndToEndTest so they are known-good. +public class LiftBackScenarioTest +{ + // three-exon contig geometry (exons 100-199, 300-399, 500-549; introns 200-299, 400-499), all-'A' genome with + // canonical GT..AG seeded at the intron boundaries so supplementary-resolve ref-verify / canonicalize have a motif to land on. + private static TestGenome scenarioGenome() + { + return new TestGenome().with(CHR_1, 2000, 'A') + .set(CHR_1, 200, "GT").set(CHR_1, 298, "AG") + .set(CHR_1, 400, "GT").set(CHR_1, 498, "AG"); + } + + private static LiftBackScenario scenario() + { + return LiftBackScenario.create() + .contig(threeExonContig()) + .genome(scenarioGenome()) + .annotatedIntron(CHR_1, 200, 299) + .annotatedIntron(CHR_1, 400, 499); + } + + @Test + public void exonSpanningReadLiftsToJunctionCigar() + { + scenario() + .read(primary("frag1", TX_CONTIG, 51, "100M")) + .read(mate("frag1", TX_CONTIG, 197, "50M")) + .run() + .assertLifted("frag1", PRIMARY, CHR_1, 150, "50M100N50M"); + } + + @Test + public void singleLocusTxMapqZeroBumpsToSixty() + { + // a tx-only read at a single genomic locus with bwa MAPQ 0 lifts and is promoted to 60 with no XA alt. + scenario() + .read(primary("frag2", TX_CONTIG, 1, "50M").mapq(0)) + .run() + .assertLifted("frag2", PRIMARY, CHR_1, 100, "50M") + .assertMapq("frag2", PRIMARY, 60) + .assertNoXa("frag2", PRIMARY); + } + + @Test + public void splitReadResolvedAcrossAnnotatedJunctionDropsSupp() + { + // primary (exon1 side) + supplementary (exon2 side) flanking the annotated intron 200-299 merge into one + // spliced primary; the supplementary is dropped, leaving primary/1 + mate/2. + scenario() + .read(primary("frag3", CHR_1, 150, "50M50S").bases("A".repeat(100))) + .read(supp("frag3", CHR_1, 300, "50S50M").bases("A".repeat(100))) + .read(mate("frag3", CHR_1, 800, "50M").bases("A".repeat(50))) + .run() + .assertLifted("frag3", PRIMARY, CHR_1, 150, "50M100N50M") + .assertSuppCount("frag3", 0); + } + + @Test + public void splitReadUniquePairBumpsMapqToSixty() + { + // bwa gave the split halves MAPQ 0; the XA alt overlaps the primary placement, so the primary+supplementary + // pair maps to a single locus and the merged spliced primary is promoted to 60. + scenario() + .read(primary("frag5", CHR_1, 150, "50M50S").mapq(0).xa(CHR_1 + ",+150,50M50S,0").bases("A".repeat(100))) + .read(supp("frag5", CHR_1, 300, "50S50M").mapq(0).bases("A".repeat(100))) + .read(mate("frag5", CHR_1, 800, "50M").bases("A".repeat(50))) + .run() + .assertLifted("frag5", PRIMARY, CHR_1, 150, "50M100N50M") + .assertSuppCount("frag5", 0) + .assertMapq("frag5", PRIMARY, 60); + } + + @Test + public void splitReadMultiLocusPairKeepsBwaMapq() + { + // same merge, but the XA alt is a distinct locus (chr1:900), so the pair does not uniquely map: the merged + // primary keeps its bwa MAPQ 0 rather than being bumped. + scenario() + .read(primary("frag6", CHR_1, 150, "50M50S").mapq(0).xa(CHR_1 + ",+900,50M50S,0").bases("A".repeat(100))) + .read(supp("frag6", CHR_1, 300, "50S50M").mapq(0).bases("A".repeat(100))) + .read(mate("frag6", CHR_1, 800, "50M").bases("A".repeat(50))) + .run() + .assertLifted("frag6", PRIMARY, CHR_1, 150, "50M100N50M") + .assertSuppCount("frag6", 0) + .assertMapq("frag6", PRIMARY, 0); + } + + @Test + public void nativeGenomicReadPassesThroughUntouched() + { + scenario() + .read(primary("frag4", CHR_1, 1000, "100M")) + .read(mate("frag4", CHR_1, 1100, "50M")) + .run() + .assertLifted("frag4", PRIMARY, CHR_1, 1000, "100M"); + } +} diff --git a/tars/src/test/java/com/hartwig/hmftools/tars/liftback/LiftBackStatsTest.java b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/LiftBackStatsTest.java new file mode 100644 index 00000000000..923c5f8a87a --- /dev/null +++ b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/LiftBackStatsTest.java @@ -0,0 +1,143 @@ +package com.hartwig.hmftools.tars.liftback; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import htsjdk.samtools.SAMFileHeader; +import htsjdk.samtools.SAMRecord; + +// covers LiftBackStats counter mechanics, composition / MAPQ tier derivation, and the summary TSV layout. +public class LiftBackStatsTest +{ + private Path mSummary; + + @Before + public void setUp() throws IOException + { + mSummary = Files.createTempFile("liftback_summary_", ".tsv"); + } + + @After + public void tearDown() throws IOException + { + Files.deleteIfExists(mSummary); + } + + private static SAMRecord newRecord(final int mapq) + { + SAMRecord record = new SAMRecord(new SAMFileHeader()); + record.setReadName("read"); + record.setMappingQuality(mapq); + return record; + } + + private static LiftBackResult resultWith(final RecordState state, + final DecidingFeature feature, final int numXaAlts, final List alignments) + { + return TarsTestFixtures.resultBuilder() + .recordState(state) + .decidingFeature(feature) + .comp(LiftBackResult.Composition.NONE) + .pos(0).cigar("*") + .inputMapq(0).updatedMapq(0) + .numXaAlts(numXaAlts).numRefAlts(0) + .numLoci(0).numDistinctCigarsAtPrimaryLocus(0) + .refFullMatch(false) + .alignments(alignments) + .build(); + } + + private static LiftedAlignment refAlignment() + { + return new LiftedAlignment( + LiftedAlignment.AlignmentSource.SELF, "1", 100, "50M", + "1", 100, "50M", + 0, 0, + null, null, null, + false, true); + } + + private static LiftedAlignment txAlignment() + { + return new LiftedAlignment( + LiftedAlignment.AlignmentSource.SELF, "txContig", 1, "50M", + "1", 100, "50M", + 0, 0, + "ENST_X", "ENSG_X", "GENEX", + false, true); + } + + @Test + public void testRecordsIncrementCountersByCategory() + { + LiftBackStats stats = new LiftBackStats(); + stats.record(newRecord(60), resultWith(RecordState.RESOLVED, DecidingFeature.SOLE_REF,0, List.of(refAlignment()))); + stats.record(newRecord(60), resultWith(RecordState.RESOLVED, DecidingFeature.SOLE_REF,0, List.of(refAlignment()))); + stats.record(newRecord(0), resultWith(RecordState.UNMAPPED, null,0, List.of())); + + assertEquals(3, stats.total()); + assertEquals(2, stats.featureCount(DecidingFeature.SOLE_REF)); + assertEquals(1, stats.stateCount(RecordState.UNMAPPED)); + } + + @Test + public void testCompositionDerivation() + { + assertEquals(LiftBackResult.Composition.NONE, + LiftBackResult.Composition.fromAlignments(List.of())); + + assertEquals(LiftBackResult.Composition.REF_ONLY, + LiftBackResult.Composition.fromAlignments(List.of(refAlignment()))); + + assertEquals(LiftBackResult.Composition.TX_ONLY, + LiftBackResult.Composition.fromAlignments(List.of(txAlignment()))); + + assertEquals(LiftBackResult.Composition.REF_AND_TX, + LiftBackResult.Composition.fromAlignments(List.of(refAlignment(), txAlignment()))); + } + + @Test + public void testMapqTierDerivation() + { + LiftBackResult result = resultWith(RecordState.RESOLVED, DecidingFeature.SOLE_REF,0, List.of(refAlignment())); + + assertEquals(LiftBackStats.MapqTier.MAPQ_ZERO, + LiftBackStats.deriveMapqTier(newRecord(0), result)); + assertEquals(LiftBackStats.MapqTier.MAPQ_POS_UNIQUE, + LiftBackStats.deriveMapqTier(newRecord(60), result)); + + LiftBackResult withAlts = resultWith(RecordState.RESOLVED, DecidingFeature.SOLE_REF,1, List.of(refAlignment())); + assertEquals(LiftBackStats.MapqTier.MAPQ_POS_MULTI, + LiftBackStats.deriveMapqTier(newRecord(60), withAlts)); + } + + @Test + public void testWriteSummaryEmitsExpectedTsvLayout() throws IOException + { + LiftBackStats stats = new LiftBackStats(); + stats.record(newRecord(60), resultWith(RecordState.RESOLVED, DecidingFeature.SOLE_REF,0, List.of(refAlignment()))); + stats.record(newRecord(0), resultWith(RecordState.UNMAPPED, null,0, List.of())); + + stats.writeSummary(mSummary.toString()); + + List lines = Files.readAllLines(mSummary); + assertEquals("Metric\tValue\tPct\tBasis", lines.get(0)); + + // flat one-metric-per-line rows; Pct is Value over the named Basis metric (blank when there is none). + assertTrue(lines.contains("records_total\t2\t\t")); + assertTrue(lines.contains("primaries_resolved\t1\t50.00\trecords_total")); + assertTrue(lines.contains("lift_failed\t0\t0.00\trecords_total")); + assertTrue(lines.contains("outcome_ref\t1\t100.00\tprimaries_resolved")); + assertTrue(lines.contains("feature_sole_ref\t1\t100.00\tprimaries_resolved")); + assertTrue(lines.contains("mapq_zero_in\t1\t100.00\tprimaries_resolved")); + } +} diff --git a/tars/src/test/java/com/hartwig/hmftools/tars/liftback/LiftBackWriterTest.java b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/LiftBackWriterTest.java new file mode 100644 index 00000000000..c7324b98f86 --- /dev/null +++ b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/LiftBackWriterTest.java @@ -0,0 +1,191 @@ +package com.hartwig.hmftools.tars.liftback; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import htsjdk.samtools.SAMRecord; + +// Verifies LiftBackWriter TSV layout: stable headers, one row per record (TSV-A) / per alignment (TSV-B). +public class LiftBackWriterTest +{ + private Path mTsvA; + private Path mTsvB; + + @Before + public void setUp() throws IOException + { + mTsvA = Files.createTempFile("liftback_a_", ".tsv"); + mTsvB = Files.createTempFile("liftback_b_", ".tsv"); + } + + @After + public void tearDown() throws IOException + { + Files.deleteIfExists(mTsvA); + Files.deleteIfExists(mTsvB); + } + + @Test + public void testHeadersAndRowsForRefOnlyResult() throws IOException + { + LiftBackResult result = TarsTestFixtures.resultBuilder() + .alignments(List.of(TarsTestFixtures.selfAlignment("1", 1000, "50M"))) + .build(); + + try(LiftBackWriter writer = new LiftBackWriter(mTsvA.toString(), mTsvB.toString())) + { + writer.write(TarsTestFixtures.primaryRecord("read1", "1", 1000, "50M"), result); + } + + List aLines = Files.readAllLines(mTsvA); + List bLines = Files.readAllLines(mTsvB); + + assertEquals(2, aLines.size()); + assertEquals(LiftBackWriter.TSV_A_HEADER_LINE, aLines.get(0)); + assertEquals("read1\t1\tPRIMARY\tRESOLVED\tREF_ONLY\tREF\tSOLE_REF\tfalse\t60\t60\t0\t1\t0\t1\t1\tfalse\tfalse\tfalse\ttrue\t1\t1000\t50M\tfalse\t\t", aLines.get(1)); + + assertEquals(2, bLines.size()); + assertEquals(LiftBackWriter.TSV_B_HEADER_LINE, bLines.get(0)); + assertEquals("read1\t1\tPRIMARY\tSELF\t1\t1000\t50M\t1\t1000\t50M\t100\t0\t\t\t", bLines.get(1)); + } + + @Test + public void testTsvBHasOneRowPerLiftedAlignment() throws IOException + { + LiftedAlignment self = new LiftedAlignment( + LiftedAlignment.AlignmentSource.SELF, "txContig", 1, "50M", + "1", 100, "50M", + 100, 0, + "ENST_X", "ENSG_X", "GENEX", + false, true); + + LiftedAlignment xa = new LiftedAlignment( + LiftedAlignment.AlignmentSource.XA_INPUT, "1", 5000, "50M", + "1", 5000, "50M", + 0, 1, + null, null, null, + false, true); + + LiftBackResult result = TarsTestFixtures.resultBuilder() + .recordState(RecordState.RESOLVED) + .decidingFeature(DecidingFeature.MULTIMAPPER) + .comp(LiftBackResult.Composition.REF_AND_TX) + .pos(100) + .inputMapq(0).updatedMapq(0) + .numXaAlts(1).numTxAlts(1) + .numLoci(2) + .geneIds("ENSG_X") + .alignments(List.of(self, xa)) + .build(); + + SAMRecord mate = TarsTestFixtures.secondMateRecord("1", 100, "50M"); + mate.setReadName("read2"); + try(LiftBackWriter writer = new LiftBackWriter(mTsvA.toString(), mTsvB.toString())) + { + writer.write(mate, result); + } + + List bLines = Files.readAllLines(mTsvB); + assertEquals(3, bLines.size()); // header + 2 alignment rows + assertTrue(bLines.get(1).startsWith("read2\t2\tPRIMARY\tSELF\t")); + assertTrue(bLines.get(2).startsWith("read2\t2\tPRIMARY\tXA_INPUT\t")); + } + + private static LiftBackResult refOnlyResult() + { + return TarsTestFixtures.resultBuilder() + .alignments(List.of(TarsTestFixtures.selfAlignment("1", 1000, "50M"))) + .build(); + } + + @Test + public void headerlessWriterEmitsDataRowsOnly() throws IOException + { + try(LiftBackWriter writer = new LiftBackWriter(mTsvA.toString(), mTsvB.toString(), false)) + { + writer.write(TarsTestFixtures.primaryRecord("read1", "1", 1000, "50M"), refOnlyResult()); + } + + List aLines = Files.readAllLines(mTsvA); + List bLines = Files.readAllLines(mTsvB); + + assertEquals(1, aLines.size()); + assertTrue(aLines.get(0).startsWith("read1\t1\tPRIMARY\t")); + assertEquals(1, bLines.size()); + assertTrue(bLines.get(0).startsWith("read1\t1\tPRIMARY\tSELF\t")); + } + + @Test + public void headerLineConstantsMatchWrittenHeaders() throws IOException + { + try(LiftBackWriter writer = new LiftBackWriter(mTsvA.toString(), mTsvB.toString())) + { + // header-only files + } + + assertEquals(Files.readAllLines(mTsvA).get(0), LiftBackWriter.TSV_A_HEADER_LINE); + assertEquals(Files.readAllLines(mTsvB).get(0), LiftBackWriter.TSV_B_HEADER_LINE); + } + + @Test + public void concatJoinsShardsUnderSingleHeader() throws IOException + { + Path shard0 = Files.createTempFile("shard0_", ".tsv"); + Path shard1 = Files.createTempFile("shard1_", ".tsv"); + Path out = Files.createTempFile("concat_", ".tsv"); + try + { + Files.write(shard0, List.of("rowA", "rowB")); + Files.write(shard1, List.of("rowC")); + + TarsApplication.concatenateTsvShards( + List.of(shard0.toString(), shard1.toString()), LiftBackWriter.TSV_A_HEADER_LINE, out.toString()); + + List lines = Files.readAllLines(out); + assertEquals(List.of(LiftBackWriter.TSV_A_HEADER_LINE, "rowA", "rowB", "rowC"), lines); + } + finally + { + Files.deleteIfExists(shard0); + Files.deleteIfExists(shard1); + Files.deleteIfExists(out); + } + } + + @Test + public void testEmptyAlignmentSetStillEmitsTsvARow() throws IOException + { + // UNMAPPED: TSV-A row written, TSV-B has header only + LiftBackResult result = TarsTestFixtures.resultBuilder() + .recordState(RecordState.UNMAPPED) + .decidingFeature(null) + .comp(LiftBackResult.Composition.NONE) + .chrom("*").pos(0).cigar("*") + .inputMapq(0).updatedMapq(0) + .numRefAlts(0) + .numLoci(0).numDistinctCigarsAtPrimaryLocus(0) + .refFullMatch(false) + .build(); + + try(LiftBackWriter writer = new LiftBackWriter(mTsvA.toString(), mTsvB.toString())) + { + writer.write(TarsTestFixtures.unpairedRecord("read3"), result); + } + + List aLines = Files.readAllLines(mTsvA); + List bLines = Files.readAllLines(mTsvB); + + assertEquals(2, aLines.size()); + assertTrue(aLines.get(1).startsWith("read3\t0\tPRIMARY\tUNMAPPED\tNONE\tUNRESOLVED\t\tfalse\t")); + assertEquals(1, bLines.size()); + } +} diff --git a/tars/src/test/java/com/hartwig/hmftools/tars/liftback/MateFieldPatcherTest.java b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/MateFieldPatcherTest.java new file mode 100644 index 00000000000..5c49310da95 --- /dev/null +++ b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/MateFieldPatcherTest.java @@ -0,0 +1,248 @@ +package com.hartwig.hmftools.tars.liftback; + +import static com.hartwig.hmftools.common.bam.SamRecordUtils.MATE_CIGAR_ATTRIBUTE; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +import htsjdk.samtools.SAMFileHeader; +import htsjdk.samtools.SAMRecord; + +public class MateFieldPatcherTest +{ + private static SAMRecord pairedMappedRecord( + final String readName, final boolean firstOfPair, + final String chromosome, final int alignmentStart, final String cigar, final boolean negStrand) + { + SAMRecord record = new SAMRecord(new SAMFileHeader()); + record.setReadName(readName); + record.setReadPairedFlag(true); + record.setFirstOfPairFlag(firstOfPair); + record.setSecondOfPairFlag(!firstOfPair); + record.setReadUnmappedFlag(false); + record.setReferenceName(chromosome); + record.setAlignmentStart(alignmentStart); + record.setCigarString(cigar); + record.setReadNegativeStrandFlag(negStrand); + record.setMappingQuality(60); + return record; + } + + private static SAMRecord pairedUnmappedRecord(final String readName, final boolean firstOfPair) + { + SAMRecord record = new SAMRecord(new SAMFileHeader()); + record.setReadName(readName); + record.setReadPairedFlag(true); + record.setFirstOfPairFlag(firstOfPair); + record.setSecondOfPairFlag(!firstOfPair); + record.setReadUnmappedFlag(true); + return record; + } + + @Test + public void testPatchSamePairSameChromSetsTlenAndMateFields() + { + LiftedMateInfoCache cache = new LiftedMateInfoCache(); + LiftedMateInfo r2Info = LiftedMateInfo.mapped("1", 400, 499, "50M", true); + cache.recordPrimaryAlignment("read1", false, r2Info); + + SAMRecord r1 = pairedMappedRecord("read1", true, "1", 100, "50M", false); + MateFieldPatcher.patchMateFields(r1, cache); + + assertEquals("1", r1.getMateReferenceName()); + assertEquals(400, r1.getMateAlignmentStart()); + assertTrue(r1.getMateNegativeStrandFlag()); + assertFalse(r1.getMateUnmappedFlag()); + // leftmost R1 at 100, rightmost end R2 at 499 → span = 400, positive on R1 + assertEquals(400, r1.getInferredInsertSize()); + // MC must reflect the partner's lifted CIGAR + assertEquals("50M", r1.getStringAttribute(MATE_CIGAR_ATTRIBUTE)); + } + + @Test + public void testMateCigarWrittenAsLiftedNCigar() + { + // junction-spanning partner: MC must carry the lifted N-CIGAR, not the stale pre-lift value + LiftedMateInfoCache cache = new LiftedMateInfoCache(); + LiftedMateInfo r2Info = LiftedMateInfo.mapped("1", 400, 999, "20M500N30M", true); + cache.recordPrimaryAlignment("read1", false, r2Info); + + SAMRecord r1 = pairedMappedRecord("read1", true, "1", 100, "50M", false); + r1.setAttribute(MATE_CIGAR_ATTRIBUTE, "50M"); // stale pre-lift MC + MateFieldPatcher.patchMateFields(r1, cache); + + assertEquals("20M500N30M", r1.getStringAttribute(MATE_CIGAR_ATTRIBUTE)); + } + + @Test + public void testMateCigarClearedWhenPartnerUnmapped() + { + LiftedMateInfoCache cache = new LiftedMateInfoCache(); + cache.recordPrimaryAlignment("read1", false, LiftedMateInfo.UNMAPPED); + + SAMRecord r1 = pairedMappedRecord("read1", true, "1", 100, "50M", false); + r1.setAttribute(MATE_CIGAR_ATTRIBUTE, "50M"); + MateFieldPatcher.patchMateFields(r1, cache); + + assertNull(r1.getStringAttribute(MATE_CIGAR_ATTRIBUTE)); + } + + @Test + public void testPatchRightmostReadGetsNegativeTlen() + { + LiftedMateInfoCache cache = new LiftedMateInfoCache(); + LiftedMateInfo r1Info = LiftedMateInfo.mapped("1", 100, 149, "50M", false); + cache.recordPrimaryAlignment("read1", true, r1Info); + + SAMRecord r2 = pairedMappedRecord("read1", false, "1", 400, "50M", true); + MateFieldPatcher.patchMateFields(r2, cache); + + // span = 449 - 100 + 1 = 350; R2 rightmost → negative TLEN + assertEquals(-350, r2.getInferredInsertSize()); + } + + @Test + public void testPatchDifferentChromsZeroesTlenAndClearsProperPair() + { + LiftedMateInfoCache cache = new LiftedMateInfoCache(); + LiftedMateInfo r2Info = LiftedMateInfo.mapped("2", 400, 499, "50M", true); + cache.recordPrimaryAlignment("read1", false, r2Info); + + SAMRecord r1 = pairedMappedRecord("read1", true, "1", 100, "50M", false); + r1.setProperPairFlag(true); + MateFieldPatcher.patchMateFields(r1, cache); + + assertEquals("2", r1.getMateReferenceName()); + assertEquals(400, r1.getMateAlignmentStart()); + assertEquals(0, r1.getInferredInsertSize()); + assertFalse(r1.getProperPairFlag()); + } + + @Test + public void testPatchWithUnmappedPartner() + { + LiftedMateInfoCache cache = new LiftedMateInfoCache(); + cache.recordPrimaryAlignment("read1", false, LiftedMateInfo.UNMAPPED); + + SAMRecord r1 = pairedMappedRecord("read1", true, "1", 100, "50M", false); + r1.setProperPairFlag(true); + MateFieldPatcher.patchMateFields(r1, cache); + + assertTrue(r1.getMateUnmappedFlag()); + // SAM: unmapped mate placed at the read's own position + assertEquals("1", r1.getMateReferenceName()); + assertEquals(100, r1.getMateAlignmentStart()); + assertEquals(0, r1.getInferredInsertSize()); + assertFalse(r1.getProperPairFlag()); + } + + @Test + public void testPatchMissingPartnerClearsProperPair() + { + LiftedMateInfoCache cache = new LiftedMateInfoCache(); + SAMRecord r1 = pairedMappedRecord("read1", true, "1", 100, "50M", false); + r1.setProperPairFlag(true); + MateFieldPatcher.patchMateFields(r1, cache); + + assertFalse(r1.getProperPairFlag()); + } + + @Test + public void testUnmappedReadWithMappedMateParkedAtMateLocus() + { + // An unmapped read whose mate is mapped is placed at the mate's coordinates so the pair stays + // together in a coord-sorted BAM. It is not a proper pair and carries no insert size. + LiftedMateInfoCache cache = new LiftedMateInfoCache(); + cache.recordPrimaryAlignment("read1", false, LiftedMateInfo.mapped("1", 400, 499, "50M", true)); + + SAMRecord r1 = pairedUnmappedRecord("read1", true); + MateFieldPatcher.patchMateFields(r1, cache); + + assertFalse(r1.getMateUnmappedFlag()); + assertEquals("1", r1.getMateReferenceName()); + assertEquals(400, r1.getMateAlignmentStart()); + assertEquals("1", r1.getReferenceName()); + assertEquals(400, r1.getAlignmentStart()); + assertEquals(0, r1.getInferredInsertSize()); + assertFalse(r1.getProperPairFlag()); + } + + @Test + public void testBothMatesUnmappedClearsOwnCoordinates() + { + // When both mates are unmapped, the read's stale pre-lift coordinates are cleared. + LiftedMateInfoCache cache = new LiftedMateInfoCache(); + cache.recordPrimaryAlignment("read1", false, LiftedMateInfo.UNMAPPED); + + SAMRecord r1 = pairedUnmappedRecord("read1", true); + r1.setReferenceName("1"); // stale pre-lift placement carried over from bwa + r1.setAlignmentStart(100); + MateFieldPatcher.patchMateFields(r1, cache); + + assertTrue(r1.getMateUnmappedFlag()); + assertEquals(SAMRecord.NO_ALIGNMENT_REFERENCE_NAME, r1.getReferenceName()); + assertEquals(SAMRecord.NO_ALIGNMENT_START, r1.getAlignmentStart()); + } + + @Test + public void testPatchUnpairedRecordIsNoOp() + { + LiftedMateInfoCache cache = new LiftedMateInfoCache(); + SAMRecord record = new SAMRecord(new SAMFileHeader()); + record.setReadName("read1"); + record.setReadPairedFlag(false); + record.setReferenceName("1"); + record.setAlignmentStart(100); + + // must not throw or attempt to read first-of-pair flag + MateFieldPatcher.patchMateFields(record, cache); + } + + @Test + public void testPatchSuppRecordPatchedFromPartnerPrimary() + { + // supplementary R1 must still use R2's primary info + LiftedMateInfoCache cache = new LiftedMateInfoCache(); + LiftedMateInfo r2Info = LiftedMateInfo.mapped("1", 400, 499, "50M", true); + cache.recordPrimaryAlignment("read1", false, r2Info); + + SAMRecord r1Supp = pairedMappedRecord("read1", true, "1", 200, "30M20S", false); + r1Supp.setSupplementaryAlignmentFlag(true); + MateFieldPatcher.patchMateFields(r1Supp, cache); + + assertEquals("1", r1Supp.getMateReferenceName()); + assertEquals(400, r1Supp.getMateAlignmentStart()); + assertTrue(r1Supp.getMateNegativeStrandFlag()); + } + + @Test + public void testPatchOverlappingPairSignFromFivePrime() + { + // Overlapping pair at same start: forward 5' (100) < reverse 5' (199), so forward read gets positive TLEN. + LiftedMateInfoCache cache = new LiftedMateInfoCache(); + cache.recordPrimaryAlignment("read1", false, LiftedMateInfo.mapped("1", 100, 199, "50M", true)); + + SAMRecord r1 = pairedMappedRecord("read1", true, "1", 100, "50M", false); + MateFieldPatcher.patchMateFields(r1, cache); + + assertEquals(100, r1.getInferredInsertSize()); + } + + @Test + public void testPatchOverlappingPairSecondOfPairForwardIsPositive() + { + // Regression: second-of-pair is the forward mate; TLEN sign must follow 5'-end position, not pair order. + LiftedMateInfoCache cache = new LiftedMateInfoCache(); + cache.recordPrimaryAlignment("read1", true, LiftedMateInfo.mapped("1", 100, 211, "39S112M", true)); + + SAMRecord r2 = pairedMappedRecord("read1", false, "1", 100, "117M34S", false); + MateFieldPatcher.patchMateFields(r2, cache); + + // forward 5'=100, reverse 5'=211 → 211 - 100 + 1 = 112, positive on forward read + assertEquals(112, r2.getInferredInsertSize()); + } +} diff --git a/tars/src/test/java/com/hartwig/hmftools/tars/liftback/SaTagRewriterTest.java b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/SaTagRewriterTest.java new file mode 100644 index 00000000000..85d0dc9adab --- /dev/null +++ b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/SaTagRewriterTest.java @@ -0,0 +1,66 @@ +package com.hartwig.hmftools.tars.liftback; + +import static com.hartwig.hmftools.common.test.GeneTestUtils.CHR_1; +import static com.hartwig.hmftools.tars.liftback.TarsTestFixtures.TX_CONTIG; +import static com.hartwig.hmftools.tars.liftback.TarsTestFixtures.threeExonContig; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import java.util.List; +import java.util.Set; + +import org.junit.Test; + +public class SaTagRewriterTest +{ + private static LiftBackResolver newResolver() + { + return new LiftBackResolver(List.of(threeExonContig())); + } + + @Test + public void testRewriteSaTag() + { + LiftBackResolver resolver = newResolver(); + + // null or empty input -> null + assertNull(SaTagRewriter.rewriteSaTag(null, resolver)); + assertNull(SaTagRewriter.rewriteSaTag("", resolver)); + + // ref-contig entry passes through unchanged + String refEntry = CHR_1 + ",1000,+,50M,60,2;"; + assertEquals(refEntry, SaTagRewriter.rewriteSaTag(refEntry, resolver)); + + // tx-contig entry lifted to genomic coordinates + assertEquals(CHR_1 + ",100,+,50M,60,2;", SaTagRewriter.rewriteSaTag(TX_CONTIG + ",1,+,50M,60,2;", resolver)); + + // a malformed entry drops the whole tag (parsing is delegated to hmf-common's SupplementaryReadData, + // which rejects the tag rather than skipping bad entries; bwa never emits malformed SA in practice) + assertNull(SaTagRewriter.rewriteSaTag("junk;" + refEntry, resolver)); + + // duplicate lifted entries deduped + assertEquals(refEntry, SaTagRewriter.rewriteSaTag(refEntry + refEntry, resolver)); + + // all entries fail -> null + assertNull(SaTagRewriter.rewriteSaTag(TX_CONTIG + ",10000,+,50M,60,0;malformed;", resolver)); + } + + @Test + public void testExcludeKeyRemovesDroppedSuppAcrossClipType() + { + LiftBackResolver resolver = newResolver(); + + // The primary's SA lists a dropped supp soft-clipped (19M247S); the exclude key (built by suppSaKey from the + // hard-clipped supp record, normalised H->S) is soft. The entry must be removed, not left dangling. + Set exclude = Set.of(CHR_1 + ":1000:-:19M247S"); + assertNull(SaTagRewriter.rewriteSaTag(CHR_1 + ",1000,-,19M247S,0,0;", resolver, exclude)); + + // and when the SA entry itself is hard-clipped, rewriteSaTag normalises H->S before matching the same key. + assertNull(SaTagRewriter.rewriteSaTag(CHR_1 + ",1000,-,19M247H,0,0;", resolver, exclude)); + + // an entry at a different locus is untouched by the exclude set + assertEquals(CHR_1 + ",2000,-,19M247S,0,0;", + SaTagRewriter.rewriteSaTag(CHR_1 + ",2000,-,19M247S,0,0;", resolver, exclude)); + } +} diff --git a/tars/src/test/java/com/hartwig/hmftools/tars/liftback/SpliceLiftBackApplyTest.java b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/SpliceLiftBackApplyTest.java new file mode 100644 index 00000000000..78234e73c37 --- /dev/null +++ b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/SpliceLiftBackApplyTest.java @@ -0,0 +1,257 @@ +package com.hartwig.hmftools.tars.liftback; + +import static com.hartwig.hmftools.common.test.GeneTestUtils.CHR_1; +import static com.hartwig.hmftools.tars.liftback.TarsTestFixtures.TX_CONTIG; +import static com.hartwig.hmftools.tars.liftback.TarsTestFixtures.assertLifted; +import static com.hartwig.hmftools.tars.liftback.TarsTestFixtures.threeExonContig; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.util.List; + +import org.junit.Test; + +import htsjdk.samtools.SAMFileHeader; +import htsjdk.samtools.SAMRecord; + +// Tests LiftBackRecordOps.applyResultToRecord: chrom/pos/CIGAR rewrite, LIFT_FAILED unmapping, and XA replacement with lifted genomic alts. +public class SpliceLiftBackApplyTest +{ + private static final String XA_TAG = "XA"; + + // Local unpaired builder kept on purpose: the shared fixture's primaryRecord sets paired/first-of-pair/proper-pair flags, + // which these apply-to-record tests must not carry. + private static SAMRecord newRecord(final String contig, final int pos, final String cigar) + { + SAMRecord record = new SAMRecord(new SAMFileHeader()); + record.setReadName("read"); + record.setReferenceName(contig); + record.setAlignmentStart(pos); + record.setCigarString(cigar); + record.setMappingQuality(60); + return record; + } + + @Test + public void testTxPrimaryRewrittenToGenomicCoords() + { + SAMRecord record = newRecord(TX_CONTIG, 51, "100M"); + LiftBackResolver resolver = new LiftBackResolver(List.of(threeExonContig())); + + LiftBackRecordOps.applyResultToRecord(record, resolver.resolve(record), new LiftedMateInfoCache()); + + assertLifted(record, CHR_1, 150, "50M100N50M"); + assertNull(record.getStringAttribute(XA_TAG)); + } + + @Test + public void testUnliftableRecordMarkedUnmappedAndStripped() + { + // pos 251 is entirely past altEnd (250) - no overhang clamp can save it -> translate fails + SAMRecord record = newRecord(TX_CONTIG, 251, "10M"); + LiftBackResolver resolver = new LiftBackResolver(List.of(threeExonContig())); + + LiftBackRecordOps.applyResultToRecord(record, resolver.resolve(record), new LiftedMateInfoCache()); + + assertTrue(record.getReadUnmappedFlag()); + assertEquals(SAMRecord.NO_ALIGNMENT_REFERENCE_NAME, record.getReferenceName()); + assertEquals(SAMRecord.NO_ALIGNMENT_START, record.getAlignmentStart()); + assertEquals(SAMRecord.NO_ALIGNMENT_CIGAR, record.getCigarString()); + assertEquals(0, record.getMappingQuality()); + assertNull(record.getStringAttribute(XA_TAG)); + } + + @Test + public void testXaTagRewrittenWithLiftedCoords() + { + SAMRecord record = newRecord(CHR_1, 1000, "50M"); + record.setAttribute(XA_TAG, TX_CONTIG + ",+1,50M,0;"); + LiftBackResolver resolver = new LiftBackResolver(List.of(threeExonContig())); + + LiftBackRecordOps.applyResultToRecord(record, resolver.resolve(record), new LiftedMateInfoCache()); + + assertEquals(CHR_1, record.getReferenceName()); + assertEquals(1000, record.getAlignmentStart()); + assertEquals(CHR_1 + ",+100,50M,0;", record.getStringAttribute(XA_TAG)); + } + + @Test + public void testUnmappedRecordLeftUntouched() + { + SAMRecord record = new SAMRecord(new SAMFileHeader()); + record.setReadName("read"); + record.setReadUnmappedFlag(true); + LiftBackResolver resolver = new LiftBackResolver(List.of(threeExonContig())); + + LiftBackRecordOps.applyResultToRecord(record, resolver.resolve(record), new LiftedMateInfoCache()); + + assertTrue(record.getReadUnmappedFlag()); + } + + // willBeUnmapped is the single source of truth for both mate-cache build and the apply-time unmap branch. + @Test + public void testWillBeUnmapped() + { + LiftBackResolver resolver = new LiftBackResolver(List.of(threeExonContig())); + + // an unliftable read is always unmapped + assertTrue(LiftBackRecordOps.willBeUnmapped(resolver.resolve(unmappedRecord()))); + + // clean unique tx read (MAPQ 60, no XA) -> kept + assertFalse(LiftBackRecordOps.willBeUnmapped(resolver.resolve(newRecord(TX_CONTIG, 51, "100M")))); + + // over the XA cap on a GENOMIC primary: MAPQ 0 with no XA -> unmapped (75+ distinct genomic loci) + SAMRecord overCap = newRecord(CHR_1, 1000, "100M"); + overCap.setMappingQuality(0); + assertTrue(LiftBackRecordOps.willBeUnmapped(resolver.resolve(overCap))); + + // a TX-CONTIG primary with MAPQ 0 + no XA hit 75+ transcript contigs of one gene -> one genomic locus, + // so the REF_ONLY-gated over-cap rule does NOT unmap it; it lifts and is kept. + SAMRecord txOverCap = newRecord(TX_CONTIG, 51, "100M"); + txOverCap.setMappingQuality(0); + assertFalse(LiftBackRecordOps.willBeUnmapped(resolver.resolve(txOverCap))); + + // MAPQ 0 but XA present = ordinary few-way multimapper -> kept + SAMRecord multimapper = newRecord(TX_CONTIG, 51, "100M"); + multimapper.setMappingQuality(0); + multimapper.setAttribute("XA", CHR_1 + ",+5000,100M,0;"); + assertFalse(LiftBackRecordOps.willBeUnmapped(resolver.resolve(multimapper))); + } + + @Test + public void testMarkPrimaryUnmappedClearsAllStaleTags() + { + SAMRecord record = newRecord(CHR_1, 1000, "50M"); + record.setReadPairedFlag(true); + record.setFirstOfPairFlag(true); + record.setProperPairFlag(true); + record.setInferredInsertSize(150); + record.setAttribute("SA", "chrX,1,+,30M,30,0;"); + record.setAttribute("XA", "chrY,+1,50M,0;"); + record.setAttribute("NH", 1); + record.setAttribute("MC", "50M"); + record.setMappingQuality(60); + + LiftBackRecordOps.markPrimaryUnmapped(record); + + assertTrue(record.getReadUnmappedFlag()); + assertEquals(SAMRecord.NO_ALIGNMENT_CIGAR, record.getCigarString()); + assertEquals(0, record.getMappingQuality()); + assertFalse(record.getProperPairFlag()); + assertEquals(0, record.getInferredInsertSize()); + assertNull(record.getStringAttribute("SA")); + assertNull(record.getStringAttribute("XA")); + assertNull(record.getIntegerAttribute("NH")); + assertNull(record.getStringAttribute("MC")); + } + + @Test + public void testTinyAnnotatedJunctionAnchorKept() + { + // tx 200 is the last base of exon2: a 1bp exon2 anchor. With ANNOTATED_JUNCTION_MIN_ANCHOR_BP=1 + // the translated junction (1M...N...M) is kept, not rolled into a softclip. This is the resolve+apply + // layer, before the engine's collapser runs - guards the floor independently of collapse. + SAMRecord record = newRecord(TX_CONTIG, 200, "50M"); + LiftBackResolver resolver = new LiftBackResolver(List.of(threeExonContig())); + + LiftBackRecordOps.applyResultToRecord(record, resolver.resolve(record), new LiftedMateInfoCache()); + + assertLifted(record, CHR_1, 399, "1M100N49M"); + } + + @Test + public void testTxPrimaryDropsRefXaAltAtSameLocus() + { + // tx primary with a softclipped ref XA at the same locus; the discriminator favours tx and drops the ref alt. + SAMRecord record = newRecord(TX_CONTIG, 51, "100M"); + record.setAttribute(XA_TAG, CHR_1 + ",+150,50M50S,0;"); + LiftBackResolver resolver = new LiftBackResolver(List.of(threeExonContig())); + + LiftBackRecordOps.applyResultToRecord(record, resolver.resolve(record), new LiftedMateInfoCache()); + + assertLifted(record, CHR_1, 150, "50M100N50M"); + assertNull(record.getStringAttribute(XA_TAG)); + } + + @Test + public void testSplicedTxRecordGetsXsAStrand() + { + // spliced tx records must carry XS:A:+/- for strand-aware junction interpretation (e.g. Isofox). + SAMRecord record = newRecord(TX_CONTIG, 51, "100M"); + LiftBackResolver resolver = new LiftBackResolver(List.of(threeExonContig())); + + LiftBackRecordOps.applyResultToRecord(record, resolver.resolve(record), new LiftedMateInfoCache()); + + assertTrue("expected N in lifted cigar", record.getCigarString().contains("N")); + assertEquals(Character.valueOf('+'), record.getAttribute("XS")); + } + + @Test + public void testNonSplicedRecordHasNoXsA() + { + // a non-spliced genomic record must NOT get XS:A. + SAMRecord record = newRecord(CHR_1, 1000, "100M"); + LiftBackResolver resolver = new LiftBackResolver(List.of(threeExonContig())); + + LiftBackRecordOps.applyResultToRecord(record, resolver.resolve(record), new LiftedMateInfoCache()); + + assertFalse("expected no N in lifted cigar", record.getCigarString().contains("N")); + assertNull(record.getAttribute("XS")); + } + + @Test + public void testStrandSwapReverseComplementsSeqAndQuals() + { + // bwa placed the read contiguously on the REVERSE strand (chr1:5000 100M); an XA tx alt crosses a junction on + // the forward strand. The discriminator promotes the spliced tx placement (JUNCTION_OVER_CONTIGUOUS), flipping + // the strand reverse->forward. SEQ/quals are stored in the reverse orientation, so applyResultToRecord must + // reverse-complement the bases and reverse the quals to keep SEQ on the genomic forward strand; otherwise the + // record is emitted mis-oriented and its recomputed NM inflates to ~read length. + SAMRecord record = newRecord(CHR_1, 5000, "100M"); + record.setReadNegativeStrandFlag(true); + record.setReadBases(("A".repeat(99) + "C").getBytes()); + byte[] quals = new byte[100]; + for(int i = 0; i < 100; ++i) + quals[i] = (byte) (i % 40); + record.setBaseQualities(quals.clone()); + record.setAttribute(XA_TAG, TX_CONTIG + ",+51,100M,0;"); + + LiftBackResolver resolver = new LiftBackResolver(List.of(threeExonContig())); + LiftBackResult result = resolver.resolve(record); + assertFalse("precondition: swap flips strand to forward", result.negativeStrand()); + + LiftBackRecordOps.applyResultToRecord(record, result, new LiftedMateInfoCache()); + + assertFalse(record.getReadNegativeStrandFlag()); + assertLifted(record, CHR_1, 150, "50M100N50M"); + assertEquals("G" + "T".repeat(99), record.getReadString()); + byte[] outQuals = record.getBaseQualities(); + assertEquals(quals[99], outQuals[0]); + assertEquals(quals[0], outQuals[99]); + } + + @Test + public void testSameStrandLiftLeavesSeqUntouched() + { + // no strand change (forward tx primary lifts forward) -> SEQ must be left exactly as-is. + SAMRecord record = newRecord(TX_CONTIG, 51, "100M"); + record.setReadBases(("A".repeat(99) + "C").getBytes()); + LiftBackResolver resolver = new LiftBackResolver(List.of(threeExonContig())); + + LiftBackRecordOps.applyResultToRecord(record, resolver.resolve(record), new LiftedMateInfoCache()); + + assertFalse(record.getReadNegativeStrandFlag()); + assertEquals("A".repeat(99) + "C", record.getReadString()); + } + + private static SAMRecord unmappedRecord() + { + SAMRecord record = new SAMRecord(new SAMFileHeader()); + record.setReadName("u"); + record.setReadUnmappedFlag(true); + return record; + } +} diff --git a/tars/src/test/java/com/hartwig/hmftools/tars/liftback/TarsTestFixtures.java b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/TarsTestFixtures.java new file mode 100644 index 00000000000..fdff0d824d4 --- /dev/null +++ b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/TarsTestFixtures.java @@ -0,0 +1,409 @@ +package com.hartwig.hmftools.tars.liftback; + +import static com.hartwig.hmftools.common.test.GeneTestUtils.CHR_1; + +import static org.junit.Assert.assertEquals; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.hartwig.hmftools.common.region.BaseRegion; +import com.hartwig.hmftools.common.test.MockRefGenome; +import com.hartwig.hmftools.tars.common.ContigEntry; +import com.hartwig.hmftools.tars.liftback.supplementary.RefSequenceSource; + +import htsjdk.samtools.SAMFileHeader; +import htsjdk.samtools.SAMRecord; + +// Shared test fixtures for the liftback suite: the standard three-exon transcript contig, SAMRecord builders, +// a MockRefGenome-backed RefSequenceSource, and factories/builder for LiftedAlignment + LiftBackResult. Centralises +// what was previously copy-pasted into a dozen test classes (and keeps the giant LiftBackResult record off call sites). +public final class TarsTestFixtures +{ + public static final String GENOMIC_CHR = CHR_1; + public static final String GENE_ID = "ENSG_TEST"; + public static final String GENE_NAME = "TESTG"; + public static final String TRANS_NAME = "ENST_TEST"; + public static final String TX_CONTIG = "ens" + GENE_ID + "_" + GENE_NAME + "_" + TRANS_NAME; + + private TarsTestFixtures() { } + + // exon spans on chr1: 100-199, 300-399, 500-549; introns 200-299, 400-499; contig length 250. + public static ContigEntry threeExonContig() + { + return new ContigEntry( + TX_CONTIG, 1, 250, GENE_ID, GENE_NAME, TRANS_NAME, GENOMIC_CHR, 1, + List.of(new BaseRegion(100, 199), new BaseRegion(300, 399), new BaseRegion(500, 549))); + } + + public static SAMRecord primaryRecord(final String contig, final int pos, final String cigar) + { + return primaryRecord("readX", contig, pos, cigar); + } + + public static SAMRecord primaryRecord(final String readName, final String contig, final int pos, final String cigar) + { + SAMRecord record = new SAMRecord(new SAMFileHeader()); + record.setReadName(readName); + record.setReferenceName(contig); + record.setAlignmentStart(pos); + record.setCigarString(cigar); + record.setMappingQuality(60); + record.setReadPairedFlag(true); + record.setFirstOfPairFlag(true); + record.setProperPairFlag(true); + return record; + } + + public static SAMRecord secondMateRecord(final String contig, final int pos, final String cigar) + { + return secondMateRecord("readX", contig, pos, cigar); + } + + public static SAMRecord secondMateRecord(final String readName, final String contig, final int pos, final String cigar) + { + SAMRecord record = primaryRecord(readName, contig, pos, cigar); + record.setFirstOfPairFlag(false); + record.setSecondOfPairFlag(true); + return record; + } + + public static SAMRecord supplementaryRecord(final String contig, final int pos, final String cigar, final String saTag) + { + SAMRecord record = supplementaryRecord("readX", contig, pos, cigar); + record.setAttribute("SA", saTag); + return record; + } + + public static SAMRecord supplementaryRecord(final String readName, final String contig, final int pos, final String cigar) + { + SAMRecord record = primaryRecord(readName, contig, pos, cigar); + record.setSupplementaryAlignmentFlag(true); + return record; + } + + public static SAMRecord unpairedRecord(final String readName) + { + SAMRecord record = new SAMRecord(new SAMFileHeader()); + record.setReadName(readName); + record.setReadPairedFlag(false); + return record; + } + + // mapped single-end (Ultima) primary: like primaryRecord but with no pairing flags set at all. + public static SAMRecord unpairedPrimaryRecord(final String contig, final int pos, final String cigar) + { + SAMRecord record = new SAMRecord(new SAMFileHeader()); + record.setReadName("readX"); + record.setReferenceName(contig); + record.setAlignmentStart(pos); + record.setCigarString(cigar); + record.setMappingQuality(60); + record.setReadPairedFlag(false); + return record; + } + + public static SAMRecord unpairedSupplementaryRecord(final String contig, final int pos, final String cigar, final String saTag) + { + SAMRecord record = unpairedPrimaryRecord(contig, pos, cigar); + record.setSupplementaryAlignmentFlag(true); + record.setAttribute("SA", saTag); + return record; + } + + // RefSequenceSource over an in-memory chromosome (1-based, inclusive); empty out-of-range -> null per contract. + public static RefSequenceSource refSource(final String chromosome, final String bases) + { + MockRefGenome ref = new MockRefGenome(true); + ref.RefGenomeMap.put(chromosome, bases); + return (chrom, posStart, posEnd) -> + { + byte[] result = ref.getBases(chrom, posStart, posEnd); + return result.length == 0 ? null : result; + }; + } + + public static byte[] repeatedBase(final int length, final char base) + { + byte[] out = new byte[length]; + Arrays.fill(out, (byte) base); + return out; + } + + // ASCII bytes for a base string, e.g. read/ref windows in the tail-extend and collapse tests. + public static byte[] bases(final String sequence) + { + return sequence.getBytes(StandardCharsets.US_ASCII); + } + + // Mutable in-memory genome for the ref-dependent passes (tail-extend / collapse / canonicalize / supplementary + // resolve ref-verify). Allocate a chromosome with with(), overwrite bases at 1-based coords with set(), then hand + // asRefSource() to the engine. Out-of-range reads return null per the RefSequenceSource contract. + public static final class TestGenome + { + private final Map mBases = new HashMap<>(); + + public TestGenome with(final String chromosome, final int length, final char fill) + { + byte[] seq = new byte[length]; + Arrays.fill(seq, (byte) fill); + mBases.put(chromosome, seq); + return this; + } + + public TestGenome with(final String chromosome, final byte[] bases) + { + mBases.put(chromosome, bases); + return this; + } + + // overwrite bases at a 1-based start (e.g. to match a read for NM=0, or seed a splice motif). + public TestGenome set(final String chromosome, final int oneBasedStart, final String sequence) + { + byte[] seq = mBases.get(chromosome); + for(int i = 0; i < sequence.length(); ++i) + { + seq[oneBasedStart - 1 + i] = (byte) sequence.charAt(i); + } + return this; + } + + // overwrite a run of one base at a 1-based start (e.g. a divergent exon/intron stretch). + public TestGenome set(final String chromosome, final int oneBasedStart, final int count, final char base) + { + byte[] seq = mBases.get(chromosome); + for(int i = 0; i < count; ++i) + { + seq[oneBasedStart - 1 + i] = (byte) base; + } + return this; + } + + public RefSequenceSource asRefSource() + { + MockRefGenome ref = new MockRefGenome(true); + for(final Map.Entry entry : mBases.entrySet()) + { + ref.RefGenomeMap.put(entry.getKey(), new String(entry.getValue(), StandardCharsets.US_ASCII)); + } + return (chrom, posStart, posEnd) -> + { + byte[] result = ref.getBases(chrom, posStart, posEnd); + return result.length == 0 ? null : result; + }; + } + } + + // assert a lifted record's genomic placement (chrom + start + cigar) in one line. + public static void assertLifted(final SAMRecord record, final String chrom, final int pos, final String cigar) + { + assertEquals(chrom, record.getReferenceName()); + assertEquals(pos, record.getAlignmentStart()); + assertEquals(cigar, record.getCigarString()); + } + + public static LiftedAlignment selfAlignment(final String chrom, final int pos, final String cigar) + { + return new LiftedAlignment( + LiftedAlignment.AlignmentSource.SELF, chrom, pos, cigar, + chrom, pos, cigar, 100, 0, null, null, null, false, true); + } + + public static ResultBuilder resultBuilder() + { + return new ResultBuilder(); + } + + // Fluent builder over the 23-component LiftBackResult record with sensible defaults (a clean single-locus + // ref-only primary). Tests override only the fields the scenario cares about. + public static final class ResultBuilder + { + private RecordState mRecordState = RecordState.RESOLVED; + private DecidingFeature mDecidingFeature = DecidingFeature.SOLE_REF; + private boolean mSwapped = false; + private LiftBackResult.Composition mComp = LiftBackResult.Composition.REF_ONLY; + private LiftBackResult.RecordRole mRole = LiftBackResult.RecordRole.PRIMARY; + private String mChrom = GENOMIC_CHR; + private int mPos = 1000; + private String mCigar = "50M"; + private boolean mNegativeStrand = false; + private boolean mHasNCigar = false; + private int mInputMapq = 60; + private int mUpdatedMapq = 60; + private int mNumXaAlts = 0; + private int mNumRefAlts = 1; + private int mNumTxAlts = 0; + private int mNumLoci = 1; + private int mNumDistinctCigarsAtPrimaryLocus = 1; + private boolean mTxHasNCigar = false; + private boolean mTxSoftClipAtBoundary = false; + private boolean mRefSoftClipped = false; + private boolean mRefFullMatch = true; + private String mGeneIds = ""; + private String mNotes = ""; + private int mTranscriptStrand = 0; + private List mAlignments = List.of(); + + public ResultBuilder recordState(final RecordState v) + { + mRecordState = v; + return this; + } + + public ResultBuilder decidingFeature(final DecidingFeature v) + { + mDecidingFeature = v; + return this; + } + + public ResultBuilder swapped(final boolean v) + { + mSwapped = v; + return this; + } + + public ResultBuilder comp(final LiftBackResult.Composition v) + { + mComp = v; + return this; + } + + public ResultBuilder role(final LiftBackResult.RecordRole v) + { + mRole = v; + return this; + } + + public ResultBuilder chrom(final String v) + { + mChrom = v; + return this; + } + + public ResultBuilder pos(final int v) + { + mPos = v; + return this; + } + + public ResultBuilder cigar(final String v) + { + mCigar = v; + return this; + } + + public ResultBuilder negativeStrand(final boolean v) + { + mNegativeStrand = v; + return this; + } + + public ResultBuilder hasNCigar(final boolean v) + { + mHasNCigar = v; + return this; + } + + public ResultBuilder inputMapq(final int v) + { + mInputMapq = v; + return this; + } + + public ResultBuilder updatedMapq(final int v) + { + mUpdatedMapq = v; + return this; + } + + public ResultBuilder numXaAlts(final int v) + { + mNumXaAlts = v; + return this; + } + + public ResultBuilder numRefAlts(final int v) + { + mNumRefAlts = v; + return this; + } + + public ResultBuilder numTxAlts(final int v) + { + mNumTxAlts = v; + return this; + } + + public ResultBuilder numLoci(final int v) + { + mNumLoci = v; + return this; + } + + public ResultBuilder numDistinctCigarsAtPrimaryLocus(final int v) + { + mNumDistinctCigarsAtPrimaryLocus = v; + return this; + } + + public ResultBuilder txHasNCigar(final boolean v) + { + mTxHasNCigar = v; + return this; + } + + public ResultBuilder txSoftClipAtBoundary(final boolean v) + { + mTxSoftClipAtBoundary = v; + return this; + } + + public ResultBuilder refSoftClipped(final boolean v) + { + mRefSoftClipped = v; + return this; + } + + public ResultBuilder refFullMatch(final boolean v) + { + mRefFullMatch = v; + return this; + } + + public ResultBuilder geneIds(final String v) + { + mGeneIds = v; + return this; + } + + public ResultBuilder notes(final String v) + { + mNotes = v; + return this; + } + + public ResultBuilder transcriptStrand(final int v) + { + mTranscriptStrand = v; + return this; + } + + public ResultBuilder alignments(final List v) + { + mAlignments = v; + return this; + } + + public LiftBackResult build() + { + return new LiftBackResult( + mRecordState, mDecidingFeature, mSwapped, mComp, mRole, mChrom, mPos, mCigar, mNegativeStrand, mHasNCigar, + mInputMapq, mUpdatedMapq, mNumXaAlts, mNumRefAlts, mNumTxAlts, mNumLoci, + mNumDistinctCigarsAtPrimaryLocus, mTxHasNCigar, mTxSoftClipAtBoundary, mRefSoftClipped, + mRefFullMatch, mGeneIds, mNotes, mTranscriptStrand, mAlignments); + } + } +} diff --git a/tars/src/test/java/com/hartwig/hmftools/tars/liftback/overhang/OverhangGateTest.java b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/overhang/OverhangGateTest.java new file mode 100644 index 00000000000..6ec333305a5 --- /dev/null +++ b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/overhang/OverhangGateTest.java @@ -0,0 +1,188 @@ +package com.hartwig.hmftools.tars.liftback.overhang; + +import static com.hartwig.hmftools.common.test.GeneTestUtils.CHR_1; +import static com.hartwig.hmftools.tars.liftback.TarsTestFixtures.bases; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import com.hartwig.hmftools.tars.liftback.TarsTestFixtures.TestGenome; +import com.hartwig.hmftools.tars.liftback.supplementary.RefSequenceSource; + +import org.junit.Test; + +// Unit tests for the overhang gate: the absolute anchor-score keep condition (> MIN_OVERHANG_SCORE), the trigger +// (softclip present or multiple junctions), the iterative multi-junction peel, and the standalone softclip reclaim. +public class OverhangGateTest +{ + private static OverhangGate gate(final TestGenome genome) + { + return new OverhangGate(genome.asRefSource()); + } + + private static TestGenome genome() + { + return new TestGenome().with(CHR_1, 5000, 'A'); + } + + @Test + public void keepsAnchorScoringAboveFive() + { + // trailing "20M 100N 6M 4S": the 6bp anchor matches its far exon (score +6 > 5), so the junction is kept. + // near exon chr1:1-20, intron 21-120, far exon anchor at chr1:121-126. + TestGenome genome = genome().set(CHR_1, 121, "CCCCCC"); + OverhangGate.Result result = gate(genome).peel(CHR_1, 1, "20M100N6M4S", bases("C".repeat(30))); + + assertEquals("20M100N6M4S", result.cigar()); + assertFalse(result.dropped()); + } + + @Test + public void collapsesAnchorScoringExactlyFive() + { + // trailing "20M 100N 5M 5S": the 5bp anchor matches its far exon (score +5), which does NOT clear the + // strict > 5 bar, so the junction collapses; the intron read-through is all 'A' vs 'C' so nothing reclaims. + TestGenome genome = genome().set(CHR_1, 121, "CCCCC"); + OverhangGate.Result result = gate(genome).peel(CHR_1, 1, "20M100N5M5S", bases("C".repeat(30))); + + assertEquals("20M10S", result.cigar()); + assertTrue(result.dropped()); + } + + @Test + public void collapsesShortAnchorOnScore() + { + // trailing "20M 100N 8M 4S": an 8bp anchor is still short (<= MIN_OVERHANG_LENGTH of 12) so it is scored - it + // mismatches its far exon (all 'A' vs 'C') so it collapses. Length alone does not save a short anchor. + OverhangGate.Result result = gate(genome()).peel(CHR_1, 1, "20M100N8M4S", bases("C".repeat(32))); + + assertEquals("20M12S", result.cigar()); + assertTrue(result.dropped()); + } + + @Test + public void trustsLongOverhangWithoutScoring() + { + // trailing "20M 100N 15M 4S": a 15bp anchor is longer than MIN_OVERHANG_LENGTH (12), so it is trusted outright + // and the junction is kept even though those bases mismatch the far exon (all 'A' vs 'C', which would score < 5). + OverhangGate.Result result = gate(genome()).peel(CHR_1, 1, "20M100N15M4S", bases("C".repeat(39))); + + assertEquals("20M100N15M4S", result.cigar()); + assertFalse(result.dropped()); + } + + @Test + public void leavesSingleJunctionNoSoftclipUntouched() + { + // "20M 100N 4M": a single junction with no adjacent softclip is Case 3 - left untouched even though the + // 4bp anchor would fail the score test if it were gated. + OverhangGate.Result result = gate(genome()).peel(CHR_1, 1, "20M100N4M", bases("C".repeat(24))); + + assertEquals("20M100N4M", result.cigar()); + assertFalse(result.dropped()); + } + + @Test + public void collapsesMultiJunctionWhenReferenceScoresBetter() + { + // "20M 50N 5M 60N 4M" (no soft clip, two junctions trigger the gate). The read is all 'A'; the terminal 4M's + // far exon (chr1:136-139) is set to 'C' so the overhang scores -16, while reading contiguously through the 60N + // (chr1:76-79) matches (+4). The intronic reference scores higher than the overhang, so the 60N collapses: the + // 4M reads into the 5M -> "20M 50N 9M". The remaining single junction has no soft clip, so the peel stops. + TestGenome genome = genome().set(CHR_1, 136, "CCCC"); + OverhangGate.Result result = gate(genome).peel(CHR_1, 1, "20M50N5M60N4M", bases("A".repeat(29))); + + assertEquals("20M50N9M", result.cigar()); + assertFalse(result.dropped()); + } + + @Test + public void keepsMultiJunctionWhenOverhangScoresPositive() + { + // "20M 50N 5M 60N 4M", read and genome all 'A': the terminal 4M aligns positively across its junction (+4 > 0), + // so the junction is kept without even checking the reference alternative. + OverhangGate.Result result = gate(genome()).peel(CHR_1, 1, "20M50N5M60N4M", bases("A".repeat(29))); + + assertEquals("20M50N5M60N4M", result.cigar()); + assertFalse(result.dropped()); + } + + @Test + public void keepsMultiJunctionWhenReferenceNotBetter() + { + // "20M 50N 5M 60N 4M", read all 'C' vs an all-'A' genome: the terminal 4M scores -16 across its junction, so + // the intronic reference is checked - but reading contiguously (chr1:76-79) also scores -16, which is not higher + // than the overhang, so the junction is kept. + OverhangGate.Result result = gate(genome()).peel(CHR_1, 1, "20M50N5M60N4M", bases("C".repeat(29))); + + assertEquals("20M50N5M60N4M", result.cigar()); + assertFalse(result.dropped()); + } + + @Test + public void keptTerminalJunctionLeavesShortInteriorExonUntouched() + { + // "20M 50N 4M 60N 40M": the terminal 40M matches (all 'A', score +40) and is kept, so the peel stops + // immediately - the short interior 4M exon flanked by two surviving junctions is never gated. + OverhangGate.Result result = gate(genome()).peel(CHR_1, 1, "20M50N4M60N40M", bases("A".repeat(64))); + + assertEquals("20M50N4M60N40M", result.cigar()); + assertFalse(result.dropped()); + } + + @Test + public void leadingJunctionCollapses() + { + // "4S 5M 100N 20M": leading anchor 5M scores +5 (not > 5) so the 100N collapses; the read-through region + // (chr1:97-105) is 'C' vs the 'A' read so nothing reclaims -> "9S 20M" starting at the near exon. + TestGenome genome = genome().set(CHR_1, 97, "CCCCCCCCC"); + OverhangGate.Result result = gate(genome).peel(CHR_1, 1, "4S5M100N20M", bases("A".repeat(29))); + + assertEquals("9S20M", result.cigar()); + assertEquals(106, result.pos()); + assertTrue(result.dropped()); + } + + @Test + public void preCollapsedResolveShape() + { + // exp8 read 25535 shape "100M 83N 3M 48S": the 3bp anchor collapses (score < 5), folding into the clip -> + // "100M 51S", supplementary resolve then merges across the true junction (see SupplementaryResolverTest). + OverhangGate.Result result = gate(genome()).peel(CHR_1, 1000, "100M83N3M48S", bases("C".repeat(151))); + + assertEquals("100M51S", result.cigar()); + assertTrue(result.dropped()); + } + + @Test + public void reclaimConvertsMatchingSoftclipPrefix() + { + // "50M 10S" whose clip continues contiguously in the genome (all 'A' vs all 'A') walks to "60M". + OverhangGate.Result result = gate(genome()).reclaimTerminalSoftClip(CHR_1, 1, "50M10S", bases("A".repeat(60))); + + assertEquals("60M", result.cigar()); + } + + @Test + public void reclaimLeavesMismatchedSoftclip() + { + // "50M 10S" whose clip bases ('C') do not match the contiguous genome ('A') is left clipped. + OverhangGate.Result result = gate(genome()) + .reclaimTerminalSoftClip(CHR_1, 1, "50M10S", bases("A".repeat(50) + "C".repeat(10))); + + assertEquals("50M10S", result.cigar()); + } + + @Test + public void disabledWithoutRefSourceIsNoOp() + { + RefSequenceSource nullSource = null; + OverhangGate gate = new OverhangGate(nullSource); + assertFalse(gate.enabled()); + + OverhangGate.Result result = gate.peel(CHR_1, 1, "20M100N5M5S", bases("C".repeat(30))); + assertEquals("20M100N5M5S", result.cigar()); + assertFalse(result.dropped()); + } +} diff --git a/tars/src/test/java/com/hartwig/hmftools/tars/liftback/supplementary/AnnotatedJunctionLoaderTest.java b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/supplementary/AnnotatedJunctionLoaderTest.java new file mode 100644 index 00000000000..685d5c25421 --- /dev/null +++ b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/supplementary/AnnotatedJunctionLoaderTest.java @@ -0,0 +1,123 @@ +package com.hartwig.hmftools.tars.liftback.supplementary; + +import static com.hartwig.hmftools.common.genome.refgenome.RefGenomeVersion.V38; +import static com.hartwig.hmftools.common.test.GeneTestUtils.addGeneData; +import static com.hartwig.hmftools.common.test.GeneTestUtils.addTransExonData; +import static com.hartwig.hmftools.common.test.GeneTestUtils.createEnsemblGeneData; +import static com.hartwig.hmftools.common.test.GeneTestUtils.createGeneDataCache; +import static com.hartwig.hmftools.common.test.GeneTestUtils.createTransExons; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.List; +import java.util.Set; + +import com.hartwig.hmftools.common.region.ChrBaseRegion; +import com.hartwig.hmftools.common.ensemblcache.EnsemblDataCache; +import com.hartwig.hmftools.common.gene.GeneData; +import com.hartwig.hmftools.common.gene.TranscriptData; + +import org.junit.Test; + +public class AnnotatedJunctionLoaderTest +{ + private static final byte POS_STRAND = 1; + private static final byte NEG_STRAND = -1; + private static final String BIOTYPE = "protein_coding"; + + private static GeneData gene(final String geneId, final String chromosome) + { + return createEnsemblGeneData(geneId, geneId, chromosome, 1, 1, 1_000_000); + } + + // exonLength 99 -> a 100-base exon spanning start..start+99 (e.g. 1000..1099). + private static TranscriptData transcript(final String geneId, final int transId, final byte strand, final int[] exonStarts) + { + return transcript(geneId, transId, strand, exonStarts, 99); + } + + private static TranscriptData transcript( + final String geneId, final int transId, final byte strand, final int[] exonStarts, final int exonLength) + { + return createTransExons(geneId, transId, strand, exonStarts, exonLength, null, null, false, BIOTYPE); + } + + @Test + public void testLoadsIntronsFromAdjacentExonPairs() + { + EnsemblDataCache cache = createGeneDataCache(); + addGeneData(cache, "1", List.of(gene("ENSG_TEST", "1"))); + + // TX_A: 3 exons -> 2 junctions; TX_B: different middle exon -> 1 distinct junction + addTransExonData(cache, "ENSG_TEST", List.of( + transcript("ENSG_TEST", 1, POS_STRAND, new int[] { 1000, 1500, 2000 }), + transcript("ENSG_TEST", 2, POS_STRAND, new int[] { 1000, 1700 }))); + + Set introns = AnnotatedJunctionLoader.deriveIntrons(cache, V38); + + assertEquals(3, introns.size()); + assertTrue(introns.contains(new ChrBaseRegion("chr1", 1100, 1499))); + assertTrue(introns.contains(new ChrBaseRegion("chr1", 1600, 1999))); + assertTrue(introns.contains(new ChrBaseRegion("chr1", 1100, 1699))); + } + + @Test + public void testChrPrefixNormalization() + { + EnsemblDataCache cache = createGeneDataCache(); + addGeneData(cache, "chr3", List.of(gene("ENSG_PREFIXED", "chr3"))); + addGeneData(cache, "3", List.of(gene("ENSG_BARE", "3"))); + + addTransExonData(cache, "ENSG_PREFIXED", List.of(transcript("ENSG_PREFIXED", 1, POS_STRAND, new int[] { 100, 500 }))); + addTransExonData(cache, "ENSG_BARE", List.of(transcript("ENSG_BARE", 2, POS_STRAND, new int[] { 1000, 1500 }))); + + Set introns = AnnotatedJunctionLoader.deriveIntrons(cache, V38); + + // both genes' introns land on chr3 regardless of the cache's chr-prefix form + assertEquals(2, introns.size()); + assertTrue(introns.contains(new ChrBaseRegion("chr3", 200, 499))); + assertTrue(introns.contains(new ChrBaseRegion("chr3", 1100, 1499))); + } + + @Test + public void testSingleExonTranscriptsProduceNoIntrons() + { + EnsemblDataCache cache = createGeneDataCache(); + addGeneData(cache, "1", List.of(gene("ENSG_X", "1"))); + addTransExonData(cache, "ENSG_X", List.of(transcript("ENSG_X", 1, POS_STRAND, new int[] { 1000 }, 1000))); + + Set introns = AnnotatedJunctionLoader.deriveIntrons(cache, V38); + + assertEquals(0, introns.size()); + } + + @Test + public void testDuplicateJunctionsAcrossTranscriptsDeduplicate() + { + EnsemblDataCache cache = createGeneDataCache(); + addGeneData(cache, "2", List.of(gene("ENSG_DUP", "2"))); + addTransExonData(cache, "ENSG_DUP", List.of( + transcript("ENSG_DUP", 1, POS_STRAND, new int[] { 100, 300 }), + transcript("ENSG_DUP", 2, POS_STRAND, new int[] { 100, 300 }))); + + Set introns = AnnotatedJunctionLoader.deriveIntrons(cache, V38); + + assertEquals(1, introns.size()); + assertTrue(introns.contains(new ChrBaseRegion("chr2", 200, 299))); + } + + @Test + public void testReverseStrandRankOrder() + { + // negative-strand transcripts have exons ranked high->low genome pos; intron coords must still be lo..hi. + EnsemblDataCache cache = createGeneDataCache(); + addGeneData(cache, "4", List.of(gene("ENSG_NEG", "4"))); + addTransExonData(cache, "ENSG_NEG", List.of(transcript("ENSG_NEG", 1, NEG_STRAND, new int[] { 1000, 2000 }))); + + Set introns = AnnotatedJunctionLoader.deriveIntrons(cache, V38); + + assertEquals(1, introns.size()); + assertTrue(introns.contains(new ChrBaseRegion("chr4", 1100, 1999))); + } +} diff --git a/tars/src/test/java/com/hartwig/hmftools/tars/liftback/supplementary/SpliceMotifTest.java b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/supplementary/SpliceMotifTest.java new file mode 100644 index 00000000000..64eb3d364de --- /dev/null +++ b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/supplementary/SpliceMotifTest.java @@ -0,0 +1,44 @@ +package com.hartwig.hmftools.tars.liftback.supplementary; + +import static com.hartwig.hmftools.tars.liftback.TarsTestFixtures.bases; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; + +public class SpliceMotifTest +{ + @Test + public void testClassifiesMotifTiers() + { + // canonical GT-AG and its reverse-complement CT-AC (strand unknown at scan time) + assertEquals(Tier.CANONICAL, SpliceMotif.classify(bases("GT"), bases("AG"))); + assertEquals(Tier.CANONICAL, SpliceMotif.classify(bases("CT"), bases("AC"))); + + // semi-canonical GC-AG / AT-AC and their reverse-complements CT-GC / GT-AT + assertEquals(Tier.SEMI_CANONICAL, SpliceMotif.classify(bases("GC"), bases("AG"))); + assertEquals(Tier.SEMI_CANONICAL, SpliceMotif.classify(bases("CT"), bases("GC"))); + assertEquals(Tier.SEMI_CANONICAL, SpliceMotif.classify(bases("AT"), bases("AC"))); + assertEquals(Tier.SEMI_CANONICAL, SpliceMotif.classify(bases("GT"), bases("AT"))); + + // lowercase normalises to the same tier + assertEquals(Tier.CANONICAL, SpliceMotif.classify(bases("gt"), bases("ag"))); + assertEquals(Tier.CANONICAL, SpliceMotif.classify(bases("ct"), bases("ac"))); + + // non-motif flanks, including a matching donor with a non-matching acceptor + assertEquals(Tier.NONE, SpliceMotif.classify(bases("AA"), bases("GG"))); + assertEquals(Tier.NONE, SpliceMotif.classify(bases("NN"), bases("NN"))); + assertEquals(Tier.NONE, SpliceMotif.classify(bases("GT"), bases("CC"))); + assertEquals(Tier.NONE, SpliceMotif.classify(bases("CC"), bases("AG"))); + } + + @Test + public void testRejectsNullOrWrongLengthInputs() + { + assertEquals(Tier.NONE, SpliceMotif.classify(null, bases("AG"))); + assertEquals(Tier.NONE, SpliceMotif.classify(bases("GT"), null)); + assertEquals(Tier.NONE, SpliceMotif.classify(bases("G"), bases("AG"))); + assertEquals(Tier.NONE, SpliceMotif.classify(bases("GT"), bases("A"))); + assertEquals(Tier.NONE, SpliceMotif.classify(bases("GTC"), bases("AG"))); + } +} diff --git a/tars/src/test/java/com/hartwig/hmftools/tars/liftback/supplementary/SupplementaryResolverTest.java b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/supplementary/SupplementaryResolverTest.java new file mode 100644 index 00000000000..273e7797dd9 --- /dev/null +++ b/tars/src/test/java/com/hartwig/hmftools/tars/liftback/supplementary/SupplementaryResolverTest.java @@ -0,0 +1,1133 @@ +package com.hartwig.hmftools.tars.liftback.supplementary; + +import static com.hartwig.hmftools.tars.liftback.TarsTestFixtures.bases; +import static com.hartwig.hmftools.tars.liftback.TarsTestFixtures.repeatedBase; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +import com.hartwig.hmftools.common.region.ChrBaseRegion; +import com.hartwig.hmftools.tars.liftback.TarsTestFixtures; +import com.hartwig.hmftools.tars.liftback.TarsTestFixtures.TestGenome; + +import org.junit.Test; + +// Tests for SupplementaryResolver. Reads are 151bp (Illumina default) unless noted. +public class SupplementaryResolverTest +{ + private static final String CHR1 = "chr1"; + private static final String CHR2 = "chr2"; + private static final int READ_LEN = 151; + + private static Set annotated(final ChrBaseRegion... introns) + { + return new HashSet<>(Arrays.asList(introns)); + } + + private static SupplementaryCandidate candidate( + final String chrom, final boolean forward, final int readLen, final int primStart, + final String primCigar, final int primMapq, final SupplementaryRecord... supps) + { + return new SupplementaryCandidate(chrom, forward, readLen, primStart, primCigar, primMapq, + supps.length == 0 ? Collections.emptyList() : Arrays.asList(supps)); + } + + private static SupplementaryRecord supp( + final int index, final String chrom, final boolean forward, final int start, + final String cigar, final int mapq) + { + return new SupplementaryRecord(index, chrom, forward, start, cigar, mapq); + } + + private SupplementaryResolver disabledResolver() + { + return new SupplementaryResolver(Collections.emptySet(), SupplementaryConfig.defaults()); + } + + private SupplementaryResolver enabledResolver(final Set annotated) + { + return new SupplementaryResolver(annotated, SupplementaryConfig.enabledDefaults()); + } + + @Test + public void testRightExtendCleanMerge() + { + // Clean complementary cigars across an annotated intron; tests core right-extend. + int primStart = 31448368; + String primCigar = "94M57S"; + int suppStart = 31448541; + String suppCigar = "94S57M"; + + ChrBaseRegion annotatedIntron = new ChrBaseRegion(CHR1, 31448462, 31448540); + SupplementaryRecord supp = supp(0, CHR1, true, suppStart, suppCigar, 60); + SupplementaryCandidate cand = candidate(CHR1, true, READ_LEN, primStart, primCigar, 255, supp); + + SupplementaryResult res = enabledResolver(annotated(annotatedIntron)).resolve(cand); + + assertTrue(res.merged()); + assertEquals("94M79N57M", res.mergedCigar()); + assertEquals(primStart, res.mergedStart()); + assertEquals(1, res.droppedSupplementaryIndices().size()); + assertEquals(Integer.valueOf(0), res.droppedSupplementaryIndices().get(0)); + assertEquals(1, res.introducedIntrons().size()); + assertEquals(annotatedIntron, res.introducedIntrons().get(0)); + assertEquals(1, res.chainDepth()); + } + + @Test + public void testResolveAcrossPreCollapsedTerminalClip() + { + // exp8 read 25535: the overhang gate now collapses the tx-contig over-run (100M83N3M48S) to 100M51S + // upstream, so supplementary resolve receives the clean clip and merges it across the true 156N junction. + // (The fold that used to do this inside the resolver was removed; the gate owns terminal micro-junction handling.) + int primStart = 1051270; + String primCigar = "100M51S"; + int suppStart = 1051525; + String suppCigar = "99S52M"; + + ChrBaseRegion annotatedIntron = new ChrBaseRegion(CHR1, 1051370, 1051525); + SupplementaryRecord supp = supp(0, CHR1, true, suppStart, suppCigar, 60); + SupplementaryCandidate cand = candidate(CHR1, true, READ_LEN, primStart, primCigar, 255, supp); + + SupplementaryResult res = enabledResolver(annotated(annotatedIntron)).resolve(cand); + + assertTrue(res.merged()); + assertEquals("100M156N51M", res.mergedCigar()); + assertEquals(primStart, res.mergedStart()); + assertEquals(annotatedIntron, res.introducedIntrons().get(0)); + } + + @Test + public void testLeftExtendCleanMerge() + { + // Mirror of right-extend: primary starts after junction, supp covers the upstream exon. + int suppStart = 31448368; + String suppCigar = "57M94S"; + int primStart = 31448541; + String primCigar = "57S94M"; + + ChrBaseRegion intron = new ChrBaseRegion(CHR1, 31448425, 31448540); + SupplementaryRecord supp = supp(0, CHR1, true, suppStart, suppCigar, 60); + SupplementaryCandidate cand = candidate(CHR1, true, READ_LEN, primStart, primCigar, 255, supp); + + SupplementaryResult res = enabledResolver(annotated(intron)).resolve(cand); + + assertTrue(res.merged()); + assertEquals("57M116N94M", res.mergedCigar()); + assertEquals(suppStart, res.mergedStart()); + } + + @Test + public void testChainMergeAcrossThreeExons() + { + // 3-exon read: primary on first exon, two supps on middle and last exons. + int primStart = 1000; + String primCigar = "50M101S"; + int suppMidStart = 2000; + String suppMidCigar = "50S60M41S"; + int suppLastStart = 3000; + String suppLastCigar = "110S41M"; + + Set set = annotated( + new ChrBaseRegion(CHR1, 1050, 1999), + new ChrBaseRegion(CHR1, 2060, 2999)); + + SupplementaryCandidate cand = candidate(CHR1, true, READ_LEN, primStart, primCigar, 255, + supp(0, CHR1, true, suppMidStart, suppMidCigar, 60), + supp(1, CHR1, true, suppLastStart, suppLastCigar, 60)); + + SupplementaryResult res = enabledResolver(set).resolve(cand); + + assertTrue(res.merged()); + assertEquals("50M950N60M940N41M", res.mergedCigar()); + assertEquals(primStart, res.mergedStart()); + assertEquals(2, res.chainDepth()); + assertEquals(2, res.droppedSupplementaryIndices().size()); + assertTrue(res.droppedSupplementaryIndices().contains(0)); + assertTrue(res.droppedSupplementaryIndices().contains(1)); + } + + @Test + public void testMergeWhenPrimaryAlreadyHasInternalN() + { + // Primary already has an internal junction; supp picks up the next exon. + int primStart = 1000; + String primCigar = "50M200N40M61S"; + int suppStart = 1500; + String suppCigar = "90S61M"; + + Set set = annotated(new ChrBaseRegion(CHR1, 1290, 1499)); + + SupplementaryCandidate cand = candidate(CHR1, true, READ_LEN, primStart, primCigar, 255, + supp(0, CHR1, true, suppStart, suppCigar, 60)); + + SupplementaryResult res = enabledResolver(set).resolve(cand); + + assertTrue(res.merged()); + assertEquals("50M200N40M210N61M", res.mergedCigar()); + } + + @Test + public void testRejectWhenDisabled() + { + SupplementaryCandidate cand = candidate(CHR1, true, READ_LEN, 1000, "94M57S", 60, + supp(0, CHR1, true, 1175, "90S61M", 60)); + + SupplementaryResult res = disabledResolver().resolve(cand); + + assertFalse(res.merged()); + } + + @Test + public void testRejectNoTerminalSoftclip() + { + SupplementaryCandidate cand = candidate(CHR1, true, READ_LEN, 1000, "151M", 60, + supp(0, CHR1, true, 2000, "90S61M", 60)); + + SupplementaryResult res = enabledResolver(annotated()).resolve(cand); + + assertFalse(res.merged()); + assertEquals(SupplementaryRejectReason.NO_TERMINAL_SOFTCLIP, res.rejectReason()); + } + + @Test + public void testRejectDifferentChromosome() + { + ChrBaseRegion annotatedIntron = new ChrBaseRegion(CHR1, 1095, 1499); + SupplementaryCandidate cand = candidate(CHR1, true, READ_LEN, 1001, "94M57S", 60, + supp(0, CHR2, true, 1500, "90S61M", 60)); + + SupplementaryResult res = enabledResolver(annotated(annotatedIntron)).resolve(cand); + + assertFalse(res.merged()); + assertEquals(SupplementaryRejectReason.DIFFERENT_CHROMOSOME, res.rejectReason()); + } + + @Test + public void testRejectOppositeStrand() + { + ChrBaseRegion annotatedIntron = new ChrBaseRegion(CHR1, 1095, 1499); + SupplementaryCandidate cand = candidate(CHR1, true, READ_LEN, 1001, "94M57S", 60, + supp(0, CHR1, false, 1500, "90S61M", 60)); + + SupplementaryResult res = enabledResolver(annotated(annotatedIntron)).resolve(cand); + + assertFalse(res.merged()); + assertEquals(SupplementaryRejectReason.OPPOSITE_STRAND, res.rejectReason()); + } + + @Test + public void testRejectIntronTooShort() + { + // Intron length 6 - below default MinIntronLength=21. + SupplementaryCandidate cand = candidate(CHR1, true, READ_LEN, 1000, "94M57S", 60, + supp(0, CHR1, true, 1100, "94S57M", 60)); + + SupplementaryResult res = enabledResolver(annotated()).resolve(cand); + + assertFalse(res.merged()); + assertEquals(SupplementaryRejectReason.INTRON_TOO_SHORT, res.rejectReason()); + } + + @Test + public void testRejectIntronTooLong() + { + SupplementaryCandidate cand = candidate(CHR1, true, READ_LEN, 1000, "94M57S", 60, + supp(0, CHR1, true, 2_001_095, "94S57M", 60)); + + SupplementaryResult res = enabledResolver(annotated()).resolve(cand); + + assertFalse(res.merged()); + assertEquals(SupplementaryRejectReason.INTRON_TOO_LONG, res.rejectReason()); + } + + @Test + public void testShortPrimaryAnchorNowMerges() + { + // Primary anchor 2bp: the MinAnchorOverhang guard was removed, so this merges on the annotated junction. + ChrBaseRegion intron = new ChrBaseRegion(CHR1, 1002, 1099); + SupplementaryCandidate cand = candidate(CHR1, true, READ_LEN, 1000, "2M149S", 60, + supp(0, CHR1, true, 1100, "2S149M", 60)); + + SupplementaryResult res = enabledResolver(annotated(intron)).resolve(cand); + + assertTrue(res.merged()); + assertEquals("2M98N149M", res.mergedCigar()); + } + + @Test + public void testShortSuppAnchorNowMerges() + { + // Supp anchor 1bp: the MinAnchorOverhang guard was removed, so this merges (trust-primary junction). + ChrBaseRegion intron = new ChrBaseRegion(CHR1, 1050, 1099); + SupplementaryCandidate cand = candidate(CHR1, true, 50, 1000, "49M1S", 60, + supp(0, CHR1, true, 1100, "49S1M", 60)); + + SupplementaryResult res = enabledResolver(annotated(intron)).resolve(cand); + + assertTrue(res.merged()); + assertEquals("49M51N1M", res.mergedCigar()); + } + + @Test + public void testRejectNovelJunctionWhenAnnotatedOnly() + { + SupplementaryConfig strict = new SupplementaryConfig(true, 21, 1_000_000, 4, true, 5, 0); + SupplementaryCandidate cand = candidate(CHR1, true, READ_LEN, 1000, "94M57S", 60, + supp(0, CHR1, true, 1200, "94S57M", 60)); + + SupplementaryResult res = new SupplementaryResolver(Collections.emptySet(), strict).resolve(cand); + + assertFalse(res.merged()); + assertEquals(SupplementaryRejectReason.NOVEL_JUNCTION, res.rejectReason()); + } + + @Test + public void testAcceptNovelJunctionWhenAnnotatedOnlyFalse() + { + SupplementaryConfig perm = new SupplementaryConfig(true, 21, 1_000_000, 4, false, 0, 0); + SupplementaryCandidate cand = candidate(CHR1, true, READ_LEN, 1000, "94M57S", 60, + supp(0, CHR1, true, 1200, "94S57M", 60)); + + SupplementaryResult res = new SupplementaryResolver(Collections.emptySet(), perm).resolve(cand); + + assertTrue(res.merged()); + assertEquals("94M106N57M", res.mergedCigar()); + } + + @Test + public void testRejectHardClipPrimary() + { + SupplementaryCandidate cand = candidate(CHR1, true, READ_LEN, 1000, "10H94M57S", 60, + supp(0, CHR1, true, 1200, "90S61M", 60)); + + SupplementaryResult res = enabledResolver(annotated()).resolve(cand); + + assertFalse(res.merged()); + assertEquals(SupplementaryRejectReason.COMPLEX_CIGAR_SHAPE, res.rejectReason()); + } + + @Test + public void testRejectHardClipSupp() + { + ChrBaseRegion intron = new ChrBaseRegion(CHR1, 1095, 1199); + SupplementaryCandidate cand = candidate(CHR1, true, READ_LEN, 1001, "94M57S", 60, + supp(0, CHR1, true, 1200, "5H90S61M", 60)); + + SupplementaryResult res = enabledResolver(annotated(intron)).resolve(cand); + + assertFalse(res.merged()); + assertEquals(SupplementaryRejectReason.COMPLEX_CIGAR_SHAPE, res.rejectReason()); + } + + @Test + public void testRejectIndelAdjacentToPrimarySoftclip() + { + ChrBaseRegion intron = new ChrBaseRegion(CHR1, 1091, 1199); + SupplementaryCandidate cand = candidate(CHR1, true, READ_LEN, 1000, "90M4I57S", 60, + supp(0, CHR1, true, 1200, "94S57M", 60)); + + SupplementaryResult res = enabledResolver(annotated(intron)).resolve(cand); + + assertFalse(res.merged()); + assertEquals(SupplementaryRejectReason.COMPLEX_CIGAR_SHAPE, res.rejectReason()); + } + + @Test + public void testRejectReadCoverageOverlap() + { + // overlap 14 bases - exceeds tolerance. + ChrBaseRegion intron = new ChrBaseRegion(CHR1, 1095, 1499); + SupplementaryCandidate cand = candidate(CHR1, true, READ_LEN, 1001, "94M57S", 60, + supp(0, CHR1, true, 1500, "80S71M", 60)); + + SupplementaryResult res = enabledResolver(annotated(intron)).resolve(cand); + + assertFalse(res.merged()); + assertEquals(SupplementaryRejectReason.READ_COVERAGE_OVERLAP, res.rejectReason()); + } + + @Test + public void testRejectReadCoverageGap() + { + ChrBaseRegion intron = new ChrBaseRegion(CHR1, 1095, 1499); + SupplementaryCandidate cand = candidate(CHR1, true, READ_LEN, 1001, "94M57S", 60, + supp(0, CHR1, true, 1500, "110S41M", 60)); + + SupplementaryResult res = enabledResolver(annotated(intron)).resolve(cand); + + assertFalse(res.merged()); + assertEquals(SupplementaryRejectReason.READ_COVERAGE_GAP, res.rejectReason()); + } + + @Test + public void testRejectRefOverlap() + { + // Supp starts upstream of primary's matched end - ref overlap. + SupplementaryCandidate cand = candidate(CHR1, true, READ_LEN, 1001, "94M57S", 60, + supp(0, CHR1, true, 1080, "94S57M", 60)); + + SupplementaryResult res = enabledResolver(annotated()).resolve(cand); + + assertFalse(res.merged()); + assertEquals(SupplementaryRejectReason.READ_COVERAGE_OVERLAP, res.rejectReason()); + } + + @Test + public void testNoSupplementaryAvailable() + { + SupplementaryCandidate cand = candidate(CHR1, true, READ_LEN, 1000, "94M57S", 60); + + SupplementaryResult res = enabledResolver(annotated()).resolve(cand); + + assertFalse(res.merged()); + assertEquals(SupplementaryRejectReason.NO_MATCHING_SUPP, res.rejectReason()); + } + + @Test + public void testRejectShapeMismatchSuppWrongClipSide() + { + SupplementaryCandidate cand = candidate(CHR1, true, READ_LEN, 1000, "94M57S", 60, + supp(0, CHR1, true, 1200, "61M90S", 60)); + + SupplementaryResult res = enabledResolver(annotated()).resolve(cand); + + assertFalse(res.merged()); + assertEquals(SupplementaryRejectReason.NO_MATCHING_SUPP, res.rejectReason()); + } + + @Test + public void testPrimaryBothSidesClippedRightExtendAccepted() + { + // Middle-anchored primary: supp past primaryRefEnd disambiguates direction → right-extend fires. + SupplementaryCandidate cand = candidate(CHR1, true, READ_LEN, 1001, "5S90M56S", 60, + supp(0, CHR1, true, 1500, "95S56M", 60)); + + SupplementaryResult res = enabledResolver(annotated()).resolve(cand); + + assertTrue(res.merged()); + assertEquals("5S90M409N56M", res.mergedCigar()); + assertEquals(1001, res.mergedStart()); + } + + @Test + public void testPrimaryBothSidesClippedLeftExtendAccepted() + { + // Mirror on the left: supp ends before primaryStart → left-extend fires. + SupplementaryCandidate cand = candidate(CHR1, true, READ_LEN, 1001, "5S90M56S", 60, + supp(0, CHR1, true, 500, "5M146S", 60)); + + SupplementaryResult res = enabledResolver(annotated()).resolve(cand); + + assertTrue(res.merged()); + assertEquals("5M496N90M56S", res.mergedCigar()); + assertEquals(500, res.mergedStart()); + } + + @Test + public void testPrimaryBothSidesClippedChainMergesBothSupps() + { + // Full middle-anchored 3-exon scenario: chain merges right first then left. + SupplementaryRecord right = supp(0, CHR1, true, 1500, "95S56M", 60); + SupplementaryRecord left = supp(1, CHR1, true, 500, "5M146S", 60); + SupplementaryCandidate cand = new SupplementaryCandidate(CHR1, true, READ_LEN, 1001, "5S90M56S", 60, + Arrays.asList(right, left)); + + SupplementaryResult res = enabledResolver(annotated()).resolve(cand); + + assertTrue(res.merged()); + assertEquals("5M496N90M409N56M", res.mergedCigar()); + assertEquals(500, res.mergedStart()); + assertEquals(2, res.chainDepth()); + } + + @Test + public void testMultipleSuppsInReachSkipsMerge() + { + // Two supps both pass every gate - refuse to guess the splice destination. + ChrBaseRegion intron = new ChrBaseRegion(CHR1, 1095, 1499); + SupplementaryCandidate cand = candidate(CHR1, true, READ_LEN, 1001, "94M57S", 60, + supp(0, CHR1, true, 1500, "94S57M", 60), + supp(1, CHR1, true, 1500, "94S57M", 60)); + + SupplementaryResult res = enabledResolver(annotated(intron)).resolve(cand); + + assertFalse(res.merged()); + assertEquals(SupplementaryRejectReason.MULTIPLE_SUPPS_IN_REACH, res.rejectReason()); + } + + @Test + public void testPrimaryMapqZeroStillMerges() + { + // MAPQ-0 primary is the common tx-contig duplicate artifact - not a disqualifier. + ChrBaseRegion intron = new ChrBaseRegion(CHR1, 31448462, 31448540); + SupplementaryCandidate cand = candidate(CHR1, true, READ_LEN, 31448368, "94M57S", 0, + supp(0, CHR1, true, 31448541, "94S57M", 60)); + + SupplementaryResult res = enabledResolver(annotated(intron)).resolve(cand); + + assertTrue(res.merged()); + } + + @Test + public void testTwoValidSuppsSameBoundaryRefusesToGuess() + { + // Two supps at different intron lengths both pass - single-supp-within-reach policy refuses to guess. + ChrBaseRegion intron1 = new ChrBaseRegion(CHR1, 1095, 1499); + ChrBaseRegion intron2 = new ChrBaseRegion(CHR1, 1095, 1799); + SupplementaryCandidate cand = candidate(CHR1, true, READ_LEN, 1001, "94M57S", 60, + supp(0, CHR1, true, 1500, "94S57M", 60), + supp(1, CHR1, true, 1800, "94S57M", 60)); + + SupplementaryResult res = enabledResolver(annotated(intron1, intron2)).resolve(cand); + + assertFalse(res.merged()); + assertEquals(SupplementaryRejectReason.MULTIPLE_SUPPS_IN_REACH, res.rejectReason()); + } + + @Test + public void testMergeWhenSuppMapqZero() + { + ChrBaseRegion intron = new ChrBaseRegion(CHR1, 1095, 1499); + SupplementaryCandidate cand = candidate(CHR1, true, READ_LEN, 1001, "94M57S", 60, + supp(0, CHR1, true, 1500, "94S57M", 0)); + + SupplementaryResult res = enabledResolver(annotated(intron)).resolve(cand); + + assertTrue(res.merged()); + } + + @Test + public void testChainDepthCap() + { + // cap=2 stops the chain after 2 merges even when more supps are available. + SupplementaryConfig cappedConfig = + new SupplementaryConfig(true, 21, 1_000_000, 2, true, 0, 0); + + int p = 1000; + SupplementaryCandidate cand = candidate(CHR1, true, READ_LEN, p, "30M121S", 60, + supp(0, CHR1, true, 2000, "30S30M91S", 60), + supp(1, CHR1, true, 3000, "60S30M61S", 60), + supp(2, CHR1, true, 4000, "90S30M31S", 60), + supp(3, CHR1, true, 5000, "120S31M", 60)); + + Set set = annotated( + new ChrBaseRegion(CHR1, 1030, 1999), + new ChrBaseRegion(CHR1, 2030, 2999), + new ChrBaseRegion(CHR1, 3030, 3999), + new ChrBaseRegion(CHR1, 4030, 4999)); + + SupplementaryResult res = new SupplementaryResolver(set, cappedConfig).resolve(cand); + + assertTrue(res.merged()); + assertEquals(2, res.chainDepth()); + assertEquals(2, res.droppedSupplementaryIndices().size()); + } + + @Test + public void testShortReadLength() + { + ChrBaseRegion intron = new ChrBaseRegion(CHR1, 1031, 1130); + SupplementaryCandidate cand = candidate(CHR1, true, 50, 1001, "30M20S", 60, + supp(0, CHR1, true, 1131, "30S20M", 60)); + + SupplementaryResult res = enabledResolver(annotated(intron)).resolve(cand); + + assertTrue(res.merged()); + assertEquals("30M100N20M", res.mergedCigar()); + } + + @Test + public void testStatisticsCounters() + { + // AnnotatedOnly=true so the novel-junction reject is observable. + SupplementaryConfig strict = new SupplementaryConfig(true, 21, 1_000_000, 4, true, 5, 0); + SupplementaryResolver resolver = new SupplementaryResolver( + annotated(new ChrBaseRegion(CHR1, 1095, 1499)), strict); + + resolver.resolve(candidate(CHR1, true, READ_LEN, 1001, "94M57S", 60, + supp(0, CHR1, true, 1500, "94S57M", 60))); + resolver.resolve(candidate(CHR1, true, READ_LEN, 1001, "94M57S", 60, + supp(0, CHR1, true, 1800, "94S57M", 60))); + resolver.resolve(candidate(CHR1, true, READ_LEN, 1001, "94M57S", 60, + supp(0, CHR2, true, 1500, "94S57M", 60))); + + SupplementaryStatistics stats = resolver.statistics(); + assertEquals(3, stats.candidatesEvaluated()); + assertEquals(1, stats.mergedTotal()); + assertEquals(1, stats.mergedAtChainDepth(1)); + assertEquals(1, stats.rejectCount(SupplementaryRejectReason.NOVEL_JUNCTION)); + assertEquals(1, stats.rejectCount(SupplementaryRejectReason.DIFFERENT_CHROMOSOME)); + } + + @Test + public void testNoMergeStillCountsCandidateAndReject() + { + SupplementaryResolver resolver = new SupplementaryResolver( + Collections.emptySet(), SupplementaryConfig.enabledDefaults()); + + SupplementaryCandidate cand = candidate(CHR1, true, READ_LEN, 1000, "94M57S", 60, + supp(0, CHR1, false, 1500, "94S57M", 60)); + + SupplementaryResult res = resolver.resolve(cand); + + assertFalse(res.merged()); + assertEquals(1, resolver.statistics().candidatesEvaluated()); + assertEquals(0, resolver.statistics().mergedTotal()); + assertEquals(1, resolver.statistics().rejectCount(SupplementaryRejectReason.OPPOSITE_STRAND)); + } + + @Test + public void testExp7Case2Chr1_31448368() + { + // exp7 case 2 (chr1:31448368): clean complementary cigars merge into the expected junction CIGAR. + ChrBaseRegion annotatedIntron = new ChrBaseRegion(CHR1, 31448462, 31448539); + SupplementaryCandidate cand = candidate(CHR1, true, READ_LEN, 31448368, "94M57S", 60, + supp(0, CHR1, true, 31448540, "94S57M", 60)); + + SupplementaryResult res = enabledResolver(annotated(annotatedIntron)).resolve(cand); + + assertTrue(res.merged()); + assertEquals("94M78N57M", res.mergedCigar()); + assertEquals(31448368, res.mergedStart()); + } + + @Test + public void testExp7Case3Chr5_34937631() + { + // exp7 chr5:34937631: overlap=2, snap picks L=61 (trust-supp) on the annotated junction, + // producing the expected 61M1166N90M. + String chr5 = "chr5"; + ChrBaseRegion annotatedIntron = new ChrBaseRegion(chr5, 34937692, 34938857); + SupplementaryCandidate cand = candidate(chr5, true, READ_LEN, 34938856, "59S92M", 60, + supp(0, chr5, true, 34937631, "61M90S", 60)); + + SupplementaryResult res = enabledResolver(annotated(annotatedIntron)).resolve(cand); + + assertTrue(res.merged()); + assertEquals("61M1166N90M", res.mergedCigar()); + assertEquals(34937631, res.mergedStart()); + } + + @Test + public void testOverlapWithinToleranceSnapsToAnnotatedJunction() + { + // overlap=4; trust-primary L=94 lands on annotated (1095, 1503). + ChrBaseRegion annotated = new ChrBaseRegion(CHR1, 1095, 1503); + SupplementaryCandidate cand = candidate(CHR1, true, 151, 1001, "94M57S", 60, + supp(0, CHR1, true, 1500, "90S61M", 60)); + + SupplementaryResult res = enabledResolver(annotated(annotated)).resolve(cand); + + assertTrue(res.merged()); + assertEquals("94M409N57M", res.mergedCigar()); + assertEquals(1001, res.mergedStart()); + assertEquals(annotated, res.introducedIntrons().get(0)); + } + + @Test + public void testOverlapWithinToleranceSnapsToAnnotatedAtTrustSuppEnd() + { + // overlap=2; trust-supp L=92 lands on annotated (1092, 1497). + ChrBaseRegion annotated = new ChrBaseRegion(CHR1, 1092, 1497); + SupplementaryCandidate cand = candidate(CHR1, true, 151, 1000, "94M57S", 60, + supp(0, CHR1, true, 1498, "92S59M", 60)); + + SupplementaryResult res = enabledResolver(annotated(annotated)).resolve(cand); + + assertTrue(res.merged()); + assertEquals("92M406N59M", res.mergedCigar()); + } + + @Test + public void testOverlapWithinToleranceSeededPickAmongAnnotated() + { + // overlap=2, both L's land on an annotated junction (a tie at the ANNOTATED tier). The pick is seeded by + // the read (deterministic), not by max-min-anchor; this candidate resolves to trustPrimary. + ChrBaseRegion trustPrimary = new ChrBaseRegion(CHR1, 1095, 1499); + ChrBaseRegion trustSupp = new ChrBaseRegion(CHR1, 1093, 1497); + SupplementaryCandidate cand = candidate(CHR1, true, 151, 1001, "94M57S", 60, + supp(0, CHR1, true, 1498, "92S59M", 60)); + + SupplementaryResult res = enabledResolver(annotated(trustPrimary, trustSupp)).resolve(cand); + + assertTrue(res.merged()); + assertEquals("94M405N57M", res.mergedCigar()); + assertEquals(trustPrimary, res.introducedIntrons().get(0)); + } + + @Test + public void testOverlapExceedsToleranceRejected() + { + ChrBaseRegion annotated = new ChrBaseRegion(CHR1, 1095, 1499); + SupplementaryCandidate cand = candidate(CHR1, true, 151, 1001, "94M57S", 60, + supp(0, CHR1, true, 1500, "88S63M", 60)); + + SupplementaryResult res = enabledResolver(annotated(annotated)).resolve(cand); + + assertFalse(res.merged()); + assertEquals(SupplementaryRejectReason.READ_COVERAGE_OVERLAP, res.rejectReason()); + } + + @Test + public void testNoAnnotatedSnapWithAnnotatedOnlyTrueRejects() + { + SupplementaryConfig strict = new SupplementaryConfig(true, 21, 1_000_000, 4, true, 5, 0); + SupplementaryCandidate cand = candidate(CHR1, true, 151, 1001, "94M57S", 60, + supp(0, CHR1, true, 1500, "92S59M", 60)); + + SupplementaryResult res = new SupplementaryResolver(Collections.emptySet(), strict).resolve(cand); + + assertFalse(res.merged()); + assertEquals(SupplementaryRejectReason.NOVEL_JUNCTION, res.rejectReason()); + } + + @Test + public void testNoAnnotatedSnapWithDefaultsFallsBackToMidpoint() + { + // AnnotatedOnly=false - with no annotated/motif snap the junction is placed at the midpoint of the + // ambiguous overlap range (rounded down), not at bwa's split point. + SupplementaryCandidate cand = candidate(CHR1, true, 151, 1001, "94M57S", 60, + supp(0, CHR1, true, 1500, "92S59M", 60)); + + SupplementaryResult res = enabledResolver(annotated()).resolve(cand); + + assertTrue(res.merged()); + assertEquals("93M407N58M", res.mergedCigar()); + } + + @Test + public void testNoAnnotatedSnapWithAnnotatedOnlyFalseFallsBackToMidpoint() + { + SupplementaryConfig perm = new SupplementaryConfig(true, 21, 1_000_000, 4, false, 5, 0); + SupplementaryCandidate cand = candidate(CHR1, true, 151, 1001, "94M57S", 60, + supp(0, CHR1, true, 1500, "92S59M", 60)); + + SupplementaryResult res = new SupplementaryResolver(Collections.emptySet(), perm).resolve(cand); + + assertTrue(res.merged()); + assertEquals("93M407N58M", res.mergedCigar()); + } + + @Test + public void testMergedIntronListReflectsChainOrder() + { + int primStart = 1000; + Set set = annotated( + new ChrBaseRegion(CHR1, 1050, 1999), + new ChrBaseRegion(CHR1, 2060, 2999)); + + SupplementaryCandidate cand = candidate(CHR1, true, READ_LEN, primStart, "50M101S", 60, + supp(0, CHR1, true, 2000, "50S60M41S", 60), + supp(1, CHR1, true, 3000, "110S41M", 60)); + + SupplementaryResult res = enabledResolver(set).resolve(cand); + + assertTrue(res.merged()); + assertEquals(2, res.introducedIntrons().size()); + assertEquals(new ChrBaseRegion(CHR1, 1050, 1999), res.introducedIntrons().get(0)); + assertEquals(new ChrBaseRegion(CHR1, 2060, 2999), res.introducedIntrons().get(1)); + } + + // Resolver wired to a base-level genome, for the motif-scan and ref-verify passes. + private static SupplementaryResolver resolverWithRef(final Set annotated, final TestGenome genome) + { + return new SupplementaryResolver( + new AnnotatedJunctionIndex(annotated), genome.asRefSource(), SupplementaryConfig.enabledDefaults()); + } + + // 'N' genome with a canonical GT-AG motif seeded at the intron flanks. + private static TestGenome refWithCanonicalIntron(final int chromLen, final int intronStart, final int intronEnd) + { + return new TestGenome().with(CHR1, chromLen, 'N') + .set(CHR1, intronStart, "GT").set(CHR1, intronEnd - 1, "AG"); + } + + // Candidate carrying read bases for ref-verify: no supps, no mate hints. + private static SupplementaryCandidate refVerifyCandidate(final int start, final String cigar, final int readLen, final byte[] readBases) + { + return new SupplementaryCandidate(CHR1, true, readLen, start, cigar, 60, + Collections.emptyList(), readBases, Collections.emptyList()); + } + + @Test + public void testMotifScanPicksCanonicalGTagWhenUnannotated() + { + // Canonical GT-AG motif placed at intron (1095, 1499) with no annotation - merge via motif scan. + SupplementaryCandidate cand = candidate(CHR1, true, 151, 1001, "94M57S", 60, + supp(0, CHR1, true, 1500, "94S57M", 60)); + + SupplementaryResult res = resolverWithRef( + Collections.emptySet(), refWithCanonicalIntron(2000, 1095, 1499)).resolve(cand); + + assertTrue(res.merged()); + assertEquals("94M405N57M", res.mergedCigar()); + assertEquals(new ChrBaseRegion(CHR1, 1095, 1499), res.introducedIntrons().get(0)); + } + + @Test + public void testMotifScanPicksAnnotatedOverMotifWhenBothPresent() + { + // overlap=2; annotated at L=94 must beat canonical motif at L=92. + ChrBaseRegion annotated = new ChrBaseRegion(CHR1, 1095, 1499); + TestGenome genome = new TestGenome().with(CHR1, 2000, 'N') + .set(CHR1, 1093, "GT").set(CHR1, 1496, "AG"); + + SupplementaryCandidate cand = candidate(CHR1, true, 151, 1001, "94M57S", 60, + supp(0, CHR1, true, 1498, "92S59M", 60)); + + SupplementaryResult res = resolverWithRef(annotated(annotated), genome).resolve(cand); + + assertTrue(res.merged()); + assertEquals("94M405N57M", res.mergedCigar()); + assertEquals(annotated, res.introducedIntrons().get(0)); + } + + @Test + public void testMotifScanPrefersCanonicalOverSemiCanonical() + { + // overlap=2; canonical (GT-AG) at L=92 must beat semi-canonical (GC-AG) at L=94. + TestGenome genome = new TestGenome().with(CHR1, 2000, 'N') + .set(CHR1, 1095, "GC").set(CHR1, 1498, "AG") // semi at (1095, 1499): GC-AG + .set(CHR1, 1093, "GT").set(CHR1, 1496, "AG"); // canonical at (1093, 1497): GT-AG + + SupplementaryCandidate cand = candidate(CHR1, true, 151, 1001, "94M57S", 60, + supp(0, CHR1, true, 1498, "92S59M", 60)); + + SupplementaryResult res = resolverWithRef(Collections.emptySet(), genome).resolve(cand); + + assertTrue(res.merged()); + assertEquals("92M405N59M", res.mergedCigar()); + assertEquals(new ChrBaseRegion(CHR1, 1093, 1497), res.introducedIntrons().get(0)); + } + + @Test + public void testMotifScanAcceptsReverseStrandCanonical() + { + // Reverse-strand transcript: genomic forward CT...AC (RC of GT-AG). Merge via motif scan. + TestGenome genome = new TestGenome().with(CHR1, 2000, 'N') + .set(CHR1, 1095, "CT").set(CHR1, 1498, "AC"); + + SupplementaryCandidate cand = candidate(CHR1, true, 151, 1001, "94M57S", 60, + supp(0, CHR1, true, 1500, "94S57M", 60)); + + SupplementaryResult res = resolverWithRef(Collections.emptySet(), genome).resolve(cand); + + assertTrue(res.merged()); + assertEquals("94M405N57M", res.mergedCigar()); + } + + @Test + public void testMotifScanIgnoredWhenNoMotifAndNoAnnotated() + { + // All-N ref, no annotation, AnnotatedOnly=false - falls through to trust-primary fallback. + TestGenome genome = new TestGenome().with(CHR1, 2000, 'N'); + SupplementaryCandidate cand = candidate(CHR1, true, 151, 1001, "94M57S", 60, + supp(0, CHR1, true, 1500, "94S57M", 60)); + + SupplementaryResult res = resolverWithRef(Collections.emptySet(), genome).resolve(cand); + + assertTrue(res.merged()); + assertEquals("94M405N57M", res.mergedCigar()); + } + + @Test + public void testMotifScanLeftExtendCanonical() + { + // Left-extend with canonical GT-AG motif at intron (1058, 1499). + TestGenome genome = new TestGenome().with(CHR1, 2000, 'N') + .set(CHR1, 1058, "GT").set(CHR1, 1498, "AG"); + + SupplementaryCandidate cand = candidate(CHR1, true, 151, 1500, "57S94M", 60, + supp(0, CHR1, true, 1001, "57M94S", 60)); + + SupplementaryResult res = resolverWithRef(Collections.emptySet(), genome).resolve(cand); + + assertTrue(res.merged()); + assertEquals("57M442N94M", res.mergedCigar()); + } + + @Test + public void testRefVerifyRightExtendSuccess() + { + // Trailing 5S "CCCCC" matches chr1:201..205 exactly across annotated intron (131, 200). + TestGenome genome = new TestGenome().with(CHR1, 300, 'A').set(CHR1, 201, 5, 'C'); + byte[] readBases = bases("A".repeat(30) + "C".repeat(5)); + + SupplementaryCandidate cand = refVerifyCandidate(101, "30M5S", 35, readBases); + + SupplementaryResult res = resolverWithRef(annotated(new ChrBaseRegion(CHR1, 131, 200)), genome).resolve(cand); + + assertTrue(res.merged()); + assertEquals("30M70N5M", res.mergedCigar()); + assertEquals(101, res.mergedStart()); + } + + @Test + public void testRefVerifyRightExtendSnapsBackOverExtendedBoundary() + { + // bwa over-extended 1 base into the intron (31M4S); boundary snap trims back to find + // the annotated intron at 131 and ref-verifies the 5-base tail. + TestGenome genome = new TestGenome().with(CHR1, 300, 'A') + .set(CHR1, 131, "C").set(CHR1, 201, 5, 'C'); + byte[] readBases = bases("A".repeat(30) + "C".repeat(5)); + + SupplementaryCandidate cand = refVerifyCandidate(101, "31M4S", 35, readBases); + + SupplementaryResult res = resolverWithRef(annotated(new ChrBaseRegion(CHR1, 131, 200)), genome).resolve(cand); + + assertTrue(res.merged()); + assertEquals("30M70N5M", res.mergedCigar()); + assertEquals(101, res.mergedStart()); + } + + @Test + public void testRefVerifyBothEndsClippedResolvesJunctionTailKeepsOtherClip() + { + // Both ends clipped: trailing 5S is the junction tail; leading 5S must survive untouched. + TestGenome genome = new TestGenome().with(CHR1, 300, 'A').set(CHR1, 201, 5, 'C'); + byte[] readBases = bases("T".repeat(5) + "A".repeat(30) + "C".repeat(5)); + + SupplementaryCandidate cand = refVerifyCandidate(101, "5S30M5S", 40, readBases); + + SupplementaryResult res = resolverWithRef(annotated(new ChrBaseRegion(CHR1, 131, 200)), genome).resolve(cand); + + assertTrue(res.merged()); + assertEquals("5S30M70N5M", res.mergedCigar()); + assertEquals(101, res.mergedStart()); + } + + @Test + public void testRefVerifyRightExtendRejectsOnMismatch() + { + // Ref all 'G'; read has 'C' - mismatch rejects the merge. + TestGenome genome = new TestGenome().with(CHR1, 300, 'G'); + byte[] readBases = bases("A".repeat(30) + "C".repeat(5)); + + SupplementaryCandidate cand = refVerifyCandidate(101, "30M5S", 35, readBases); + + SupplementaryResult res = resolverWithRef(annotated(new ChrBaseRegion(CHR1, 131, 200)), genome).resolve(cand); + + assertFalse(res.merged()); + assertEquals(SupplementaryRejectReason.REF_VERIFY_MISMATCH_TOO_HIGH, res.rejectReason()); + } + + @Test + public void testRefVerifyClipsMismatchNotRecoveredByScore() + { + // 20-base trailing clip: 18 proximal matches, then a mismatch with only 1 trailing match. Under the + // shared bwa-mem score walk (mismatch -4) one trailing match cannot recover the mismatch, so the run + // stops at 18 and the outer 2 bases stay soft-clipped. + TestGenome genome = new TestGenome().with(CHR1, 300, 'A') + .set(CHR1, 201, 18, 'C').set(CHR1, 219, "GC"); + byte[] readBases = bases("A".repeat(20) + "C".repeat(20)); + + SupplementaryCandidate cand = refVerifyCandidate(101, "20M20S", 40, readBases); + + SupplementaryResult res = resolverWithRef(annotated(new ChrBaseRegion(CHR1, 121, 200)), genome).resolve(cand); + + assertTrue(res.merged()); + assertEquals("20M80N18M2S", res.mergedCigar()); + } + + @Test + public void testRefVerifyLeftExtendSuccess() + { + // Leading 5S "TTTTT" matches chr1:126..130 exactly across annotated intron (131, 200). + TestGenome genome = new TestGenome().with(CHR1, 300, 'A').set(CHR1, 126, 5, 'T'); + byte[] readBases = bases("T".repeat(5) + "A".repeat(30)); + + SupplementaryCandidate cand = refVerifyCandidate(201, "5S30M", 35, readBases); + + SupplementaryResult res = resolverWithRef(annotated(new ChrBaseRegion(CHR1, 131, 200)), genome).resolve(cand); + + assertTrue(res.merged()); + assertEquals("5M70N30M", res.mergedCigar()); + assertEquals(126, res.mergedStart()); + } + + @Test + public void testRefVerifyRejectsWhenNoCandidateExon() + { + // Empty annotation - no candidate exon for the trailing softclip. + TestGenome genome = new TestGenome().with(CHR1, 300, 'A'); + SupplementaryCandidate cand = refVerifyCandidate(101, "30M5S", 35, repeatedBase(35, 'A')); + + SupplementaryResult res = resolverWithRef(Collections.emptySet(), genome).resolve(cand); + + assertFalse(res.merged()); + assertEquals(SupplementaryRejectReason.REF_VERIFY_NO_CANDIDATE_EXON, res.rejectReason()); + } + + @Test + public void testRefVerifyAmbiguousRejected() + { + // Two annotated introns share donor; both downstream exons (201..205 and 501..505) match "CCCCC" - ambiguous. + TestGenome genome = new TestGenome().with(CHR1, 600, 'A') + .set(CHR1, 201, 5, 'C').set(CHR1, 501, 5, 'C'); + byte[] readBases = bases("A".repeat(30) + "C".repeat(5)); + + SupplementaryCandidate cand = refVerifyCandidate(101, "30M5S", 35, readBases); + + SupplementaryResult res = resolverWithRef( + annotated(new ChrBaseRegion(CHR1, 131, 200), new ChrBaseRegion(CHR1, 131, 500)), genome).resolve(cand); + + assertFalse(res.merged()); + assertEquals(SupplementaryRejectReason.REF_VERIFY_AMBIGUOUS, res.rejectReason()); + } + + @Test + public void testRefVerifyWithoutRefSourceSkipsSilently() + { + // No RefSequenceSource - ref-verify skips silently. + SupplementaryCandidate cand = candidate(CHR1, true, 35, 101, "30M5S", 60); + + SupplementaryResult res = enabledResolver(annotated()).resolve(cand); + + assertFalse(res.merged()); + assertEquals(SupplementaryRejectReason.NO_MATCHING_SUPP, res.rejectReason()); + } + + @Test + public void testRefVerifyPartialMatchTrailingKeepsOuterClip() + { + // 15 proximal bases match the exon; outer 4 are adapter residual - only proximal bases convert + // to M, outer stay clipped: 30M19S → 30M70N15M4S. + TestGenome genome = new TestGenome().with(CHR1, 300, 'A').set(CHR1, 201, 15, 'C'); // downstream exon: 15 bases + byte[] readBases = bases("A".repeat(30) + "C".repeat(15) + "T".repeat(4)); // overhang + adapter residual + + SupplementaryCandidate cand = refVerifyCandidate(101, "30M19S", 49, readBases); + + SupplementaryResult res = resolverWithRef(annotated(new ChrBaseRegion(CHR1, 131, 200)), genome).resolve(cand); + + assertTrue(res.merged()); + assertEquals("30M70N15M4S", res.mergedCigar()); + assertEquals(101, res.mergedStart()); + } + + @Test + public void testRefVerifyPartialMatchLeadingKeepsOuterClip() + { + // Mirror on leading side: outer 4 are adapter, proximal 15 are real overhang → 4S15M70N30M. + TestGenome genome = new TestGenome().with(CHR1, 300, 'A').set(CHR1, 116, 15, 'C'); // upstream exon: 15 bases + byte[] readBases = bases("T".repeat(4) + "C".repeat(15) + "A".repeat(30)); // adapter + overhang + + SupplementaryCandidate cand = refVerifyCandidate(201, "19S30M", 49, readBases); + + SupplementaryResult res = resolverWithRef(annotated(new ChrBaseRegion(CHR1, 131, 200)), genome).resolve(cand); + + assertTrue(res.merged()); + assertEquals("4S15M70N30M", res.mergedCigar()); + assertEquals(116, res.mergedStart()); + } + + @Test + public void testRefVerifyShortPartialRunNowMerges() + { + // 8-base proximal match: the MinPartialMatchRun guard was removed, so the partial match now merges with + // the divergent residual left soft-clipped (30M70N8M11S). + TestGenome genome = new TestGenome().with(CHR1, 300, 'A').set(CHR1, 201, 8, 'C'); // only 8 matching bases + byte[] readBases = bases("A".repeat(30) + "C".repeat(8) + "T".repeat(11)); // 8-base match + divergent residual + + SupplementaryCandidate cand = refVerifyCandidate(101, "30M19S", 49, readBases); + + SupplementaryResult res = resolverWithRef(annotated(new ChrBaseRegion(CHR1, 131, 200)), genome).resolve(cand); + + assertTrue(res.merged()); + assertEquals("30M70N8M11S", res.mergedCigar()); + assertEquals(101, res.mergedStart()); + } + + @Test + public void testRefVerifySnapsBackSevenBaseOverExtension() + { + // bwa over-extended 7 bases into the intron (37M3S); boundary snap (MaxBoundaryShift≥8) + // trims back to find annotated intron 131 and re-verifies the 10-base tail. + TestGenome genome = new TestGenome().with(CHR1, 300, 'A') + .set(CHR1, 131, 7, 'C') // bwa's 7 over-extended bases + .set(CHR1, 201, 10, 'C'); // real downstream exon + byte[] readBases = bases("A".repeat(30) + "C".repeat(10)); // 7 over-extended + 3 clipped + + SupplementaryCandidate cand = refVerifyCandidate(101, "37M3S", 40, readBases); + + SupplementaryResult res = resolverWithRef(annotated(new ChrBaseRegion(CHR1, 131, 200)), genome).resolve(cand); + + assertTrue(res.merged()); + assertEquals("30M70N10M", res.mergedCigar()); + assertEquals(101, res.mergedStart()); + } + + @Test + public void testMateHintUsedAsFallbackWhenAnnotationMisses() + { + // Mate hint overrides the midpoint fallback when no annotation is within the snap window. + SupplementaryCandidate withoutHint = candidate(CHR1, true, 151, 1001, "94M57S", 60, + supp(0, CHR1, true, 1498, "92S59M", 60)); + SupplementaryResult resNoHint = enabledResolver(annotated()).resolve(withoutHint); + assertTrue(resNoHint.merged()); + assertEquals(1094, resNoHint.introducedIntrons().get(0).start()); + + ChrBaseRegion hint = new ChrBaseRegion(CHR1, 1093, 1497); + SupplementaryCandidate withHint = new SupplementaryCandidate( + CHR1, true, 151, 1001, "94M57S", 60, + Arrays.asList(supp(0, CHR1, true, 1498, "92S59M", 60)), + null, Arrays.asList(hint)); + SupplementaryResult resHinted = enabledResolver(annotated()).resolve(withHint); + assertTrue(resHinted.merged()); + assertEquals(1093, resHinted.introducedIntrons().get(0).start()); + assertEquals("92M405N59M", resHinted.mergedCigar()); + } + + @Test + public void testMateHintIgnoredWhenAnnotatedMatchExists() + { + // Annotated junction beats mate hint when both are within the snap window. + ChrBaseRegion annotatedJunc = new ChrBaseRegion(CHR1, 1095, 1499); + ChrBaseRegion hint = new ChrBaseRegion(CHR1, 1093, 1497); + SupplementaryCandidate cand = new SupplementaryCandidate( + CHR1, true, 151, 1001, "94M57S", 60, + Arrays.asList(supp(0, CHR1, true, 1498, "92S59M", 60)), + null, Arrays.asList(hint)); + + SupplementaryResult res = enabledResolver(annotated(annotatedJunc)).resolve(cand); + + assertTrue(res.merged()); + assertEquals(1095, res.introducedIntrons().get(0).start()); + } + + @Test + public void testMateHintOutsideOverlapWindowIgnored() + { + // Hint outside the overlap window - ignored, falls back to the midpoint of the range. + ChrBaseRegion hint = new ChrBaseRegion(CHR1, 50000, 50100); + SupplementaryCandidate cand = new SupplementaryCandidate( + CHR1, true, 151, 1001, "94M57S", 60, + Arrays.asList(supp(0, CHR1, true, 1498, "92S59M", 60)), + null, Arrays.asList(hint)); + + SupplementaryResult res = enabledResolver(annotated()).resolve(cand); + + assertTrue(res.merged()); + assertEquals(1094, res.introducedIntrons().get(0).start()); + } + + @Test + public void testRejectReasonIsNullOnSuccess() + { + ChrBaseRegion intron = new ChrBaseRegion(CHR1, 1095, 1499); + SupplementaryCandidate cand = candidate(CHR1, true, READ_LEN, 1001, "94M57S", 60, + supp(0, CHR1, true, 1500, "94S57M", 60)); + + SupplementaryResult res = enabledResolver(annotated(intron)).resolve(cand); + + assertTrue(res.merged()); + assertNull(res.rejectReason()); + } +}