upgrade to 0.5.2

Illumina · Sep 13, 2023 · ac2d5f4 · ac2d5f4
1 parent e32af48
commit ac2d5f4
Show file tree

Hide file tree

Showing 157 changed files with 28,057 additions and 2,008 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,13 +1,13 @@
-FROM mcr.microsoft.com/dotnet/sdk:5.0 as builder
+FROM mcr.microsoft.com/dotnet/sdk:6.0 as builder
 WORKDIR /src
 COPY . /src
 RUN cd Wittyer \
-    && dotnet publish -f netcoreapp2.0 -r linux-x64 -c Release -o /output \
+    && dotnet publish -f net6.0 -r linux-x64 -c Release -o /output \
     && chmod +x /output/Wittyer
 
-FROM mcr.microsoft.com/dotnet/runtime:5.0
+FROM mcr.microsoft.com/dotnet/runtime:6.0
 LABEL git_repository=https://git.illumina.com/DASTE/Ilmn.Das.App.Wittyer.git
 WORKDIR /opt/Wittyer
-RUN apt-get -y update && apt-get -y install tabix libunwind8
+RUN apt-get -y update && apt-get -y install tabix libunwind8 openssl
 COPY --from=builder /output /opt/Wittyer
 ENTRYPOINT ["/opt/Wittyer/Wittyer"]
diff --git a/Ilmn.Das.App.Wittyer.Test/BorderDistanceTest.cs b/Ilmn.Das.App.Wittyer.Test/BorderDistanceTest.cs
@@ -1,13 +1,13 @@
 using System;
 using System.Collections.Generic;
 using System.Collections.Immutable;
+using Ilmn.Das.App.Wittyer.Utilities.Enums;
+using Ilmn.Das.App.Wittyer.Vcf.Readers;
 using Ilmn.Das.App.Wittyer.Vcf.Variants;
 using Ilmn.Das.Core.Tries.Extensions;
 using Ilmn.Das.Std.BioinformaticUtils.Genomes;
-using Ilmn.Das.Std.VariantUtils.Vcf.Parsers;
-using Ilmn.Das.Std.VariantUtils.Vcf.Variants;
+
 using Ilmn.Das.Std.XunitUtils;
-using JetBrains.Annotations;
 using Xunit;
 
 namespace Ilmn.Das.App.Wittyer.Test
@@ -40,7 +40,7 @@ public static class BorderDistanceTest
 
         private const double PercentDistance = 0.05;
 
-        private static readonly IReadOnlyList<uint> Bins = ImmutableList.Create(1000U, 10000U);
+        private static readonly IReadOnlyList<(uint, bool)> Bins = ImmutableList.Create((1000U, false), (10000U, false));
 
         private const uint BasepairDistance = 500;
 
@@ -86,24 +86,24 @@ public static void WorksWithBreakend()
             MultiAssert.AssertAll();
         }
 
-        [NotNull]
-        private static IWittyerSimpleVariant CreateWittyerVariant([NotNull] string vcfline)
+        private static IWittyerSimpleVariant CreateWittyerVariant(string vcfline)
         {
             var baseVariant = VcfVariant.TryParse(vcfline,
                     VcfVariantParserSettings.Create(ImmutableList.Create("normal"), GenomeAssembly.Grch37))
                 .GetOrThrowDebug();
-            WittyerType.ParseFromVariant(baseVariant, true, null, out var svType);
+            var failed = WittyerType.ParseFromVariant(baseVariant, null, out var wittyerType, out _);
+
+            if (failed != FailedReason.Unset)
+                throw new NotSupportedException($"This method failed: {failed}");
 
-            if (svType == null)
-                throw new NotSupportedException("This method does not handle svType null");
 
             return WittyerVariantInternal
-                .Create(baseVariant, baseVariant.Samples["normal"], svType, Bins, PercentDistance, BasepairDistance);
+                .Create(baseVariant, baseVariant.Samples["normal"], wittyerType, Bins, PercentDistance,
+                    BasepairDistance, 0);
         }
 
 
-        [NotNull]
-        private static IWittyerBnd CreateWittyerBnd([NotNull] string bndLine1, [NotNull] string bndLine2)
+        private static IWittyerBnd CreateWittyerBnd(string bndLine1, string bndLine2)
         {
             var variant = VcfVariant.TryParse(bndLine1,
                     VcfVariantParserSettings.Create(ImmutableList.Create("normal"), GenomeAssembly.Grch37))

diff --git a/Ilmn.Das.App.Wittyer.Test/ConfigParsingTest.cs b/Ilmn.Das.App.Wittyer.Test/ConfigParsingTest.cs
@@ -24,7 +24,7 @@ public class ConfigParsingTest
             "Path 'variantType', line 2, position 16.";
 
         [Theory]
-        [InlineData("Config-missing-field.json", "Setting for variant type 'Deletion' did not contain required fields: bpDistance.")]
+        [InlineData("Config-missing-field.json", "Setting for variant type 'Deletion' did not contain required fields: absoluteThreshold.")]
         [InlineData("Config-duplicate-sv-type.json", "Duplicate variant type 'Deletion' in the config file.")]
         [InlineData("Config-unrecognized-sv-type.json", "Unknown variant type 'Unrecognizable' in the config file.")]
         [InlineData("Config-unrecognized-field.json", "Unrecognized field names in config file: unrecognizedField.")]

diff --git a/Ilmn.Das.App.Wittyer.Test/EndToEndTest.cs b/Ilmn.Das.App.Wittyer.Test/EndToEndTest.cs
@@ -11,6 +11,7 @@
 using Ilmn.Das.App.Wittyer.Stats;
 using Ilmn.Das.App.Wittyer.Utilities;
 using Ilmn.Das.App.Wittyer.Utilities.Enums;
+using Ilmn.Das.App.Wittyer.Vcf.Readers;
 using Ilmn.Das.App.Wittyer.Vcf.Variants;
 using Ilmn.Das.Core.Tries.Extensions;
 using Ilmn.Das.Std.AppUtils.Collections;
@@ -19,11 +20,7 @@
 using Ilmn.Das.Std.BioinformaticUtils.Contigs;
 using Ilmn.Das.Std.BioinformaticUtils.GenomicFeatures;
 using Ilmn.Das.Std.VariantUtils.Vcf;
-using Ilmn.Das.Std.VariantUtils.Vcf.Parsers;
-using Ilmn.Das.Std.VariantUtils.Vcf.Readers;
-using Ilmn.Das.Std.VariantUtils.Vcf.Variants;
 using Ilmn.Das.Std.XunitUtils;
-using JetBrains.Annotations;
 using Monad.Parsec;
 using Newtonsoft.Json;
 using Xunit;
@@ -34,7 +31,7 @@ namespace Ilmn.Das.App.Wittyer.Test
     public class EndToEndTest
     {
         private static readonly IImmutableDictionary<WittyerType, InputSpec> InputSpecs =
-            InputSpec.GenerateCustomInputSpecs(true, WittyerType.AllTypes, percentDistance: 0.05)
+            InputSpec.GenerateCustomInputSpecs(true, WittyerType.AllTypes, percentThreshold: 0.05)
                 .ToImmutableDictionary(x => x.VariantType, x => x);
 
         //CNV files
@@ -80,11 +77,12 @@ public void SvWorksWithDefault()
 
             var outputDirectory = Path.GetRandomFileName().ToDirectoryInfo();
             var wittyerSettings = WittyerSettings.Create(outputDirectory, GermlineTruth, GermlineQuery,
-                ImmutableList<ISamplePair>.Empty, EvaluationMode.Default, InputSpecs);
+                ImmutableList<ISamplePair>.Empty, EvaluationMode.GenotypeMatching, InputSpecs);
 
-            var json = MainLauncher.GenerateJson(wittyerSettings, MainLauncher.GenerateResults(wittyerSettings).EnumerateSuccesses(), EmptyCmd);
+            var json = MainLauncher.GenerateJson(wittyerSettings, 
+                MainLauncher.GenerateResults(wittyerSettings).EnumerateSuccesses(), EmptyCmd)[(MatchEnum.Unmatched, true)];
             //var str = JsonConvert.SerializeObject(json, Formatting.Indented);
-            var stats = json.GetOrThrow().PerSampleStats.First();
+            var stats = json.PerSampleStats.First();
             var expectedStats = JsonConvert.DeserializeObject<GeneralStats>(File.ReadAllText(SvJsonGt.FullName))
                 .PerSampleStats.First();
 
@@ -134,7 +132,7 @@ public void CnvWorksWithCrossType()
 
             var stats = MainLauncher
                 .GenerateJson(wittyerSettings, results,
-                    EmptyCmd).GetOrThrow().PerSampleStats.First();
+                    EmptyCmd)[(MatchEnum.Unmatched, false)].PerSampleStats.First();
 
             // make sure to check for null
             MultiAssert.True(stats.QuerySampleName != null);
@@ -155,13 +153,13 @@ public void CnvWorksWithCrossType()
             MultiAssert.AssertAll();
         }
 
-        private static bool ParseVariantGetTag([NotNull] string variant, WitDecision targetDecision)
+        private static bool ParseVariantGetTag(string variant, WitDecision targetDecision)
         {
             var parser = VcfVariantParserSettings.Create(new List<string> { "sample" });
             var vcfVariant = VcfVariant.TryParse(variant, parser).GetOrThrow();
             var sampleDictionary = vcfVariant.Samples.Single().Value.SampleDictionary;
             if (vcfVariant.Info.TryGetValue(WittyerConstants.WittyerMetaInfoLineKeys.Who, out var who))
-                if (who.Split(WittyerConstants.SampleValueDel).Length > WittyerConstants.MaxNumberOfAnnotations)
+                if (who.Split(WittyerConstants.SampleValueDel).Length > WittyerConstants.DefaultMaxMatches)
                     return false;
             return sampleDictionary
                 .TryGetValue(WittyerConstants.WittyerMetaInfoLineKeys.Wit).Any(it =>
@@ -182,7 +180,7 @@ public void CnvWorksWithSimpleCounting()
 
             var actualStats = MainLauncher
                 .GenerateJson(wittyerSettings, MainLauncher.GenerateResults(wittyerSettings).EnumerateSuccesses(),
-                    EmptyCmd).GetOrThrow().PerSampleStats.First();
+                    EmptyCmd)[(MatchEnum.Unmatched, false)].PerSampleStats.First();
             //var str = JsonConvert.SerializeObject(actualStats, Formatting.Indented);
             var jsonText = File.ReadAllText(CnvJsonSc.FullName);
             var expectedStats = JsonConvert.DeserializeObject<GeneralStats>(jsonText)
@@ -398,7 +396,7 @@ bool IsCountedCnv(IVcfVariant variant)
                     if (!variant.Info.TryGetValue(VcfConstants.SvTypeKey, out var svType)
                         || !WittyerConstants.BaseLevelStatsTypeStrings.Contains(svType))
                         return false;
-                    if (!int.TryParse(cn, out var ploidy))
+                    if (!decimal.TryParse(cn, out var ploidy))
                         return false;
 
                     isRef = (hasGt
@@ -450,8 +448,8 @@ public void SvWorksWithSimpleCount()
 
             //var insertionSpec = InputSpecs[WittyerVariantType.Insertion];
             var insertionSpec = InputSpec.Create(WittyerType.Insertion,
-                WittyerConstants.DefaultBins.SetItem(0, (50, false)),
-                WittyerConstants.DefaultBpOverlap, 0.05, WittyerConstants.DefaultExcludeFilters,
+                WittyerType.Insertion.DefaultBins.SetItem(0, (50, false)),
+                WittyerConstants.DefaultAbsThreshold, 0.05, WittyerConstants.DefaultExcludeFilters,
                 WittyerConstants.DefaultIncludeFilters, null);
             var outputDirectory = Path.GetRandomFileName().ToDirectoryInfo();
             var wittyerSettings = WittyerSettings.Create(outputDirectory, SomaticTruth,
@@ -471,8 +469,8 @@ public void SvWorksWithSimpleCount()
             testStrings = WittyerVcfWriter.GenerateVcfStrings(null, truth, null).Where(line => !line.StartsWith(VcfConstants.Header.Prefix));
             MultiAssert.True(testStrings.All(s => ParseVariantGetTag(s, WitDecision.FalseNegative)));
 
-            var actualStats = GeneralStats.Create(results,
-                    wittyerSettings.Mode == EvaluationMode.Default, wittyerSettings.InputSpecs, EmptyCmd).PerSampleStats
+            var actualStats = GeneralStats.Create(results, wittyerSettings.InputSpecs, 
+                    wittyerSettings.Mode == EvaluationMode.CrossTypeAndSimpleCounting,EmptyCmd)[(MatchEnum.Unmatched, false)].PerSampleStats
                 .First();
             var expectedStats = JsonConvert.DeserializeObject<GeneralStats>(File.ReadAllText(SvJsonSc.FullName))
                 .PerSampleStats.First();

diff --git a/Ilmn.Das.App.Wittyer.Test/Ilmn.Das.App.Wittyer.Test.csproj b/Ilmn.Das.App.Wittyer.Test/Ilmn.Das.App.Wittyer.Test.csproj
@@ -1,15 +1,16 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netcoreapp2.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
+    <Nullable>enable</Nullable>
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|AnyCPU'">
-    <LangVersion>7.2</LangVersion>
+    <LangVersion>default</LangVersion>
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
-    <LangVersion>7.2</LangVersion>
+    <LangVersion>default</LangVersion>
   </PropertyGroup>
 
   <ItemGroup>
@@ -74,6 +75,51 @@
     <None Update="Resources\WIT-154\truth.vcf">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
+    <None Update="Resources\WIT-155\bed.bed">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="Resources\WIT-155\config.json">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="Resources\WIT-155\query.vcf">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="Resources\VNTR\catalog1.vcf">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="Resources\VNTR\catalog2.vcf">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="Resources\VNTR\HG002.delphi0.1.GRCh38.SV.vcf.gz">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="Resources\VNTR\HG002.delphi0.1.GRCh38.TandemRepeatRegions.bed.gz">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="Resources\VNTR\HG002.delphi0.1.GRCh38.VNTR.vcf">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="Resources\VNTR\HG002.delphi0.1.GRCh38.SV.vcf">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="Resources\VNTR\HG002.delphi0.1.GRCh38.SV.vcf.gz.tbi">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="Resources\VNTR\HG002.delphi0.1.GRCh38.TandemRepeatRegions.bed.gz.tbi">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="Resources\VNTR\HG002.delphi0.1.GRCh38.VNTR.vcf.gz">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="Resources\VNTR\HG002.delphi0.1.GRCh38.TandemRepeatRegions.bed">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="Resources\WIT-173\query.vcf">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="Resources\WIT-173\truth.vcf">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
   </ItemGroup>
 
   <ItemGroup>

diff --git a/Ilmn.Das.App.Wittyer.Test/IntervalTreeUtilsTest.cs b/Ilmn.Das.App.Wittyer.Test/IntervalTreeUtilsTest.cs
@@ -5,7 +5,6 @@
 using Ilmn.Das.Std.AppUtils.Misc;
 using Ilmn.Das.Std.BioinformaticUtils.Bed;
 using Ilmn.Das.Std.XunitUtils;
-using JetBrains.Annotations;
 using Xunit;
 
 namespace Ilmn.Das.App.Wittyer.Test
@@ -38,7 +37,7 @@ public static void MergeMultipleIntervalWorks()
         }
 
         public static TheoryData<IInterval<uint>, IEnumerable<IInterval<uint>>> SubstractIntervalData 
-            = new TheoryData<IInterval<uint>, IEnumerable<IInterval<uint>>>
+            = new()
             {
                 {new InclusiveInterval<uint>(12, 13),
                     new IInterval<uint>[]{new ClosedOpenInterval<uint>(10, 12), new OpenClosedInterval<uint>(13, 15)} },
@@ -102,7 +101,6 @@ public static void GetTotalLengthWorks()
         }
 
 
-        [NotNull]
         private static IntervalTree<uint, IInterval<uint>> GetOriginalTree()
         {
             var tree = new IntervalTree<uint, IInterval<uint>>

diff --git a/Ilmn.Das.App.Wittyer.Test/MergedIntervalTreeTest.cs b/Ilmn.Das.App.Wittyer.Test/MergedIntervalTreeTest.cs
@@ -1,4 +1,4 @@
-using System.Collections.Generic;
+using System.Collections.Generic;
 using System.Linq;
 using Ilmn.Das.App.Wittyer.Utilities;
 using Ilmn.Das.Std.AppUtils.Intervals;