Upgrade to spec version 3

exercism · May 6, 2021 · 1cd34d8 · 1cd34d8
1 parent ae1ccfa
commit 1cd34d8
Show file tree

Hide file tree

Showing 44 changed files with 261 additions and 15 deletions.
diff --git a/src/Exercism.TestRunner.CSharp/TestResultParser.cs b/src/Exercism.TestRunner.CSharp/TestResultParser.cs
@@ -50,6 +50,7 @@ private static TestResult ToTestResult(XmlUnitTestResult xmlUnitTestResult, Meth
                 Status = xmlUnitTestResult.Status(),
                 Message = xmlUnitTestResult.Message(),
                 Output = xmlUnitTestResult.Output(),
+                TaskId = testMethodDeclaration.TaskId(),
                 TestCode = testMethodDeclaration.TestCode()
             };
 
@@ -94,6 +95,18 @@ private static string TestCode(this MethodDeclarationSyntax testMethod)
                 .WithoutLeadingTrivia()
                 .ToString();
         }
+
+        private static int? TaskId(this MethodDeclarationSyntax testMethod) =>
+            testMethod.AttributeLists
+                .SelectMany(attributeList => attributeList.Attributes)
+                .Where(attribute =>
+                    attribute.Name.ToString() == "Task" &&
+                    attribute.ArgumentList != null &&
+                    attribute.ArgumentList.Arguments.Count == 1 &&
+                    attribute.ArgumentList.Arguments[0].Expression.IsKind(SyntaxKind.NumericLiteralExpression))
+                .Select(attribute => (LiteralExpressionSyntax)attribute.ArgumentList.Arguments[0].Expression)
+                .Select(taskNumberExpression => (int?)taskNumberExpression.Token.Value!)
+                .FirstOrDefault();
     }
 
     [XmlRoot(ElementName = "Output", Namespace = "http://microsoft.com/schemas/VisualStudio/TeamTest/2010")]

diff --git a/src/Exercism.TestRunner.CSharp/TestRun.cs b/src/Exercism.TestRunner.CSharp/TestRun.cs
@@ -16,6 +16,9 @@ internal class TestResult
 
         [JsonPropertyName("status")]
         public TestStatus Status { get; set; }
+
+        [JsonPropertyName("task_id")]
+        public int? TaskId { get; set; }
 
         [JsonPropertyName("message")]
         public string Message { get; set; }
@@ -30,7 +33,7 @@ internal class TestResult
     internal class TestRun
     {
         [JsonPropertyName("version")]
-        public int Version { get; set; } = 2;
+        public int Version { get; set; } = 3;
 
         [JsonPropertyName("status")]
         public TestStatus Status { get; set; }

diff --git a/test/Exercism.TestRunner.CSharp.IntegrationTests/Solutions/AllTestsWithTask/Fake.cs b/test/Exercism.TestRunner.CSharp.IntegrationTests/Solutions/AllTestsWithTask/Fake.cs
@@ -0,0 +1,8 @@
+public static class Fake
+{
+    public static int Add(int x, int y) => x + y;
+
+    public static int Sub(int x, int y) => x - y;
+
+    public static int Mul(int x, int y) => x * y;
+}
diff --git a/test/Exercism.TestRunner.CSharp.IntegrationTests/Solutions/AllTestsWithTask/Fake.csproj b/test/Exercism.TestRunner.CSharp.IntegrationTests/Solutions/AllTestsWithTask/Fake.csproj
@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net5.0</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Compile Remove="Example.cs" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.8.0" />
+    <PackageReference Include="xunit" Version="2.4.1" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3" />
+    <PackageReference Include="Exercism.Tests" Version="0.1.0-alpha" />
+  </ItemGroup>
+
+</Project>
diff --git a/test/Exercism.TestRunner.CSharp.IntegrationTests/Solutions/AllTestsWithTask/FakeTests.cs b/test/Exercism.TestRunner.CSharp.IntegrationTests/Solutions/AllTestsWithTask/FakeTests.cs
@@ -0,0 +1,17 @@
+using Xunit;
+using Exercism.Tests;
+
+public class FakeTests
+{
+    [Fact]
+    [Task(1)]
+    public void Add_should_add_numbers() => Assert.Equal(2, Fake.Add(1, 1));
+
+    [Fact]
+    [Task(2)]
+    public void Sub_should_subtract_numbers() => Assert.Equal(4, Fake.Sub(7, 3));
+
+    [Fact]
+    [Task(3)]
+    public void Mul_should_multiply_numbers() => Assert.Equal(6, Fake.Mul(2, 3));
+}
diff --git a/...rcism.TestRunner.CSharp.IntegrationTests/Solutions/AllTestsWithTask/expected_results.json b/...rcism.TestRunner.CSharp.IntegrationTests/Solutions/AllTestsWithTask/expected_results.json
@@ -0,0 +1,24 @@
+{
+  "version": 3,
+  "status": "pass",
+  "tests": [
+    {
+      "name": "Add should add numbers",
+      "status": "pass",
+      "task_id": 1,
+      "test_code": "Assert.Equal(2, Fake.Add(1, 1))"
+    },
+    {
+      "name": "Sub should subtract numbers",
+      "status": "pass",
+      "task_id": 2,
+      "test_code": "Assert.Equal(1, Fake.Sub(2, 1))"
+    },
+    {
+      "name": "Mul should multiply numbers",
+      "status": "pass",
+      "task_id": 3,
+      "test_code": "Assert.Equal(5, Fake.Mul(2, 3))"
+    }
+  ]
+}
diff --git a/test/Exercism.TestRunner.CSharp.IntegrationTests/Solutions/MultipleCompileErrors/Fake.csproj b/test/Exercism.TestRunner.CSharp.IntegrationTests/Solutions/MultipleCompileErrors/Fake.csproj
@@ -12,6 +12,7 @@
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.8.0" />
     <PackageReference Include="xunit" Version="2.4.1" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3" />
+    <PackageReference Include="Exercism.Tests" Version="0.1.0-alpha" />
   </ItemGroup>
 
 </Project>
diff --git a/....TestRunner.CSharp.IntegrationTests/Solutions/MultipleCompileErrors/expected_results.json b/....TestRunner.CSharp.IntegrationTests/Solutions/MultipleCompileErrors/expected_results.json
@@ -1,5 +1,5 @@
 {
-  "version": 2,
+  "version": 3,
   "status": "error",
   "message": "FakeTests.cs(5,66): error CS0117: \u0027Fake\u0027 does not contain a definition for \u0027Add\u0027\nFakeTests.cs(7,66): error CS0117: \u0027Fake\u0027 does not contain a definition for \u0027Sub\u0027",
   "tests": []

diff --git a/...TestRunner.CSharp.IntegrationTests/Solutions/MultipleTestClassesWithAllPasses/Fake.csproj b/...TestRunner.CSharp.IntegrationTests/Solutions/MultipleTestClassesWithAllPasses/Fake.csproj
@@ -12,6 +12,7 @@
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.8.0" />
     <PackageReference Include="xunit" Version="2.4.1" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3" />
+    <PackageReference Include="Exercism.Tests" Version="0.1.0-alpha" />
   </ItemGroup>
 
 </Project>
diff --git a/....CSharp.IntegrationTests/Solutions/MultipleTestClassesWithAllPasses/expected_results.json b/....CSharp.IntegrationTests/Solutions/MultipleTestClassesWithAllPasses/expected_results.json
@@ -1,5 +1,5 @@
 {
-  "version": 2,
+  "version": 3,
   "status": "pass",
   "tests": [
     {

diff --git a/...rcism.TestRunner.CSharp.IntegrationTests/Solutions/MultipleTestsWithAllPasses/Fake.csproj b/...rcism.TestRunner.CSharp.IntegrationTests/Solutions/MultipleTestsWithAllPasses/Fake.csproj
@@ -12,6 +12,7 @@
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.8.0" />
     <PackageReference Include="xunit" Version="2.4.1" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3" />
+    <PackageReference Include="Exercism.Tests" Version="0.1.0-alpha" />
   </ItemGroup>
 
 </Project>
diff --git a/...Runner.CSharp.IntegrationTests/Solutions/MultipleTestsWithAllPasses/expected_results.json b/...Runner.CSharp.IntegrationTests/Solutions/MultipleTestsWithAllPasses/expected_results.json
@@ -1,5 +1,5 @@
 {
-  "version": 2,
+  "version": 3,
   "status": "pass",
   "tests": [
     {

diff --git a/...m.TestRunner.CSharp.IntegrationTests/Solutions/MultipleTestsWithMultipleFails/Fake.csproj b/...m.TestRunner.CSharp.IntegrationTests/Solutions/MultipleTestsWithMultipleFails/Fake.csproj
@@ -12,6 +12,7 @@
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.8.0" />
     <PackageReference Include="xunit" Version="2.4.1" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3" />
+    <PackageReference Include="Exercism.Tests" Version="0.1.0-alpha" />
   </ItemGroup>
 
 </Project>
diff --git a/...er.CSharp.IntegrationTests/Solutions/MultipleTestsWithMultipleFails/expected_results.json b/...er.CSharp.IntegrationTests/Solutions/MultipleTestsWithMultipleFails/expected_results.json
@@ -1,5 +1,5 @@
 {
-  "version": 2,
+  "version": 3,
   "status": "fail",
   "tests": [
     {

diff --git a/...cism.TestRunner.CSharp.IntegrationTests/Solutions/MultipleTestsWithSingleFail/Fake.csproj b/...cism.TestRunner.CSharp.IntegrationTests/Solutions/MultipleTestsWithSingleFail/Fake.csproj
@@ -12,6 +12,7 @@
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.8.0" />
     <PackageReference Include="xunit" Version="2.4.1" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3" />
+    <PackageReference Include="Exercism.Tests" Version="0.1.0-alpha" />
   </ItemGroup>
 
 </Project>
diff --git a/...unner.CSharp.IntegrationTests/Solutions/MultipleTestsWithSingleFail/expected_results.json b/...unner.CSharp.IntegrationTests/Solutions/MultipleTestsWithSingleFail/expected_results.json
@@ -1,5 +1,5 @@
 {
-  "version": 2,
+  "version": 3,
   "status": "fail",
   "tests": [
     {

diff --git a/...cism.TestRunner.CSharp.IntegrationTests/Solutions/MultipleTestsWithTestOutput/Fake.csproj b/...cism.TestRunner.CSharp.IntegrationTests/Solutions/MultipleTestsWithTestOutput/Fake.csproj
@@ -12,6 +12,7 @@
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.8.0" />
     <PackageReference Include="xunit" Version="2.4.1" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3" />
+    <PackageReference Include="Exercism.Tests" Version="0.1.0-alpha" />
   </ItemGroup>
 
 </Project>
diff --git a/...unner.CSharp.IntegrationTests/Solutions/MultipleTestsWithTestOutput/expected_results.json b/...unner.CSharp.IntegrationTests/Solutions/MultipleTestsWithTestOutput/expected_results.json
@@ -1,5 +1,5 @@
 {
-  "version": 2,
+  "version": 3,
   "status": "fail",
   "tests": [
     {

diff --git a/...r.CSharp.IntegrationTests/Solutions/MultipleTestsWithTestOutputExceedingLimit/Fake.csproj b/...r.CSharp.IntegrationTests/Solutions/MultipleTestsWithTestOutputExceedingLimit/Fake.csproj
@@ -12,6 +12,7 @@
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.8.0" />
     <PackageReference Include="xunit" Version="2.4.1" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3" />
+    <PackageReference Include="Exercism.Tests" Version="0.1.0-alpha" />
   </ItemGroup>
 
 </Project>
diff --git a/...ntegrationTests/Solutions/MultipleTestsWithTestOutputExceedingLimit/expected_results.json b/...ntegrationTests/Solutions/MultipleTestsWithTestOutputExceedingLimit/expected_results.json
@@ -1,5 +1,5 @@
 {
-  "version": 2,
+  "version": 3,
   "status": "fail",
   "tests": [
     {

diff --git a/test/Exercism.TestRunner.CSharp.IntegrationTests/Solutions/NoTasks/Fake.cs b/test/Exercism.TestRunner.CSharp.IntegrationTests/Solutions/NoTasks/Fake.cs
@@ -0,0 +1,8 @@
+public static class Fake
+{
+    public static int Add(int x, int y) => x + y;
+
+    public static int Sub(int x, int y) => x - y;
+
+    public static int Mul(int x, int y) => x * y;
+}
diff --git a/test/Exercism.TestRunner.CSharp.IntegrationTests/Solutions/NoTasks/Fake.csproj b/test/Exercism.TestRunner.CSharp.IntegrationTests/Solutions/NoTasks/Fake.csproj
@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net5.0</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Compile Remove="Example.cs" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.8.0" />
+    <PackageReference Include="xunit" Version="2.4.1" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3" />
+    <PackageReference Include="Exercism.Tests" Version="0.1.0-alpha" />
+  </ItemGroup>
+
+</Project>
diff --git a/test/Exercism.TestRunner.CSharp.IntegrationTests/Solutions/NoTasks/FakeTests.cs b/test/Exercism.TestRunner.CSharp.IntegrationTests/Solutions/NoTasks/FakeTests.cs
@@ -0,0 +1,10 @@
+using Xunit;
+public class FakeTests
+{
+    [Fact]
+    public void Add_should_add_numbers() => Assert.Equal(2, Fake.Add(1, 1));
+    [Fact]
+    public void Sub_should_subtract_numbers() => Assert.Equal(4, Fake.Sub(7, 3));
+    [Fact]
+    public void Mul_should_multiply_numbers() => Assert.Equal(6, Fake.Mul(2, 3));
+}
diff --git a/test/Exercism.TestRunner.CSharp.IntegrationTests/Solutions/NoTasks/expected_results.json b/test/Exercism.TestRunner.CSharp.IntegrationTests/Solutions/NoTasks/expected_results.json
@@ -0,0 +1,21 @@
+{
+  "version": 3,
+  "status": "pass",
+  "tests": [
+    {
+      "name": "Add should add numbers",
+      "status": "pass",
+      "test_code": "Assert.Equal(2, Fake.Add(1, 1))"
+    },
+    {
+      "name": "Sub should subtract numbers",
+      "status": "pass",
+      "test_code": "Assert.Equal(1, Fake.Sub(2, 1))"
+    },
+    {
+      "name": "Mul should multiply numbers",
+      "status": "pass",
+      "test_code": "Assert.Equal(5, Fake.Mul(2, 3))"
+    }
+  ]
+}
diff --git a/test/Exercism.TestRunner.CSharp.IntegrationTests/Solutions/NoTests/Fake.csproj b/test/Exercism.TestRunner.CSharp.IntegrationTests/Solutions/NoTests/Fake.csproj
@@ -12,6 +12,7 @@
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.8.0" />
     <PackageReference Include="xunit" Version="2.4.1" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3" />
+    <PackageReference Include="Exercism.Tests" Version="0.1.0-alpha" />
   </ItemGroup>
 
 </Project>
diff --git a/test/Exercism.TestRunner.CSharp.IntegrationTests/Solutions/NoTests/expected_results.json b/test/Exercism.TestRunner.CSharp.IntegrationTests/Solutions/NoTests/expected_results.json
@@ -1,5 +1,5 @@
 {
-  "version": 2,
+  "version": 3,
   "status": "error",
   "tests": []
 }
diff --git a/test/Exercism.TestRunner.CSharp.IntegrationTests/Solutions/NotImplemented/Fake.csproj b/test/Exercism.TestRunner.CSharp.IntegrationTests/Solutions/NotImplemented/Fake.csproj
@@ -12,6 +12,7 @@
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.8.0" />
     <PackageReference Include="xunit" Version="2.4.1" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3" />
+    <PackageReference Include="Exercism.Tests" Version="0.1.0-alpha" />
   </ItemGroup>
 
 </Project>
diff --git a/...xercism.TestRunner.CSharp.IntegrationTests/Solutions/NotImplemented/expected_results.json b/...xercism.TestRunner.CSharp.IntegrationTests/Solutions/NotImplemented/expected_results.json
@@ -1,5 +1,5 @@
 {
-  "version": 2,
+  "version": 3,
   "status": "fail",
   "tests": [
     {

diff --git a/test/Exercism.TestRunner.CSharp.IntegrationTests/Solutions/SingleCompileError/Fake.csproj b/test/Exercism.TestRunner.CSharp.IntegrationTests/Solutions/SingleCompileError/Fake.csproj
@@ -12,6 +12,7 @@
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.8.0" />
     <PackageReference Include="xunit" Version="2.4.1" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3" />
+    <PackageReference Include="Exercism.Tests" Version="0.1.0-alpha" />
   </ItemGroup>
 
 </Project>
diff --git a/...ism.TestRunner.CSharp.IntegrationTests/Solutions/SingleCompileError/expected_results.json b/...ism.TestRunner.CSharp.IntegrationTests/Solutions/SingleCompileError/expected_results.json
@@ -1,5 +1,5 @@
 {
-  "version": 2,
+  "version": 3,
   "status": "error",
   "message": "Fake.cs(3,49): error CS1002: ; expected",
   "tests": []

diff --git a/test/Exercism.TestRunner.CSharp.IntegrationTests/Solutions/SingleTestThatFails/Fake.csproj b/test/Exercism.TestRunner.CSharp.IntegrationTests/Solutions/SingleTestThatFails/Fake.csproj
@@ -12,6 +12,7 @@
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.8.0" />
     <PackageReference Include="xunit" Version="2.4.1" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3" />
+    <PackageReference Include="Exercism.Tests" Version="0.1.0-alpha" />
   </ItemGroup>
 
 </Project>
diff --git a/...sm.TestRunner.CSharp.IntegrationTests/Solutions/SingleTestThatFails/expected_results.json b/...sm.TestRunner.CSharp.IntegrationTests/Solutions/SingleTestThatFails/expected_results.json
@@ -1,5 +1,5 @@
 {
-  "version": 2,
+  "version": 3,
   "status": "fail",
   "tests": [
     {

diff --git a/test/Exercism.TestRunner.CSharp.IntegrationTests/Solutions/SingleTestThatPasses/Fake.csproj b/test/Exercism.TestRunner.CSharp.IntegrationTests/Solutions/SingleTestThatPasses/Fake.csproj
@@ -12,6 +12,7 @@
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.8.0" />
     <PackageReference Include="xunit" Version="2.4.1" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3" />
+    <PackageReference Include="Exercism.Tests" Version="0.1.0-alpha" />
   </ItemGroup>
 
 </Project>
diff --git a/...m.TestRunner.CSharp.IntegrationTests/Solutions/SingleTestThatPasses/expected_results.json b/...m.TestRunner.CSharp.IntegrationTests/Solutions/SingleTestThatPasses/expected_results.json
@@ -1,5 +1,5 @@
 {
-  "version": 2,
+  "version": 3,
   "status": "pass",
   "tests": [
     {

diff --git a/...Runner.CSharp.IntegrationTests/Solutions/SingleTestThatPassesWithDifferentSlug/Foo.csproj b/...Runner.CSharp.IntegrationTests/Solutions/SingleTestThatPassesWithDifferentSlug/Foo.csproj
@@ -12,6 +12,7 @@
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.8.0" />
     <PackageReference Include="xunit" Version="2.4.1" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3" />
+    <PackageReference Include="Exercism.Tests" Version="0.1.0-alpha" />
   </ItemGroup>
 
 </Project>
diff --git a/...rp.IntegrationTests/Solutions/SingleTestThatPassesWithDifferentSlug/expected_results.json b/...rp.IntegrationTests/Solutions/SingleTestThatPassesWithDifferentSlug/expected_results.json
@@ -1,5 +1,5 @@
 {
-  "version": 2,
+  "version": 3,
   "status": "pass",
   "tests": [
     {