Update e2e tests and e2e test infrastructure (#414)

2023-04-20 14:42:27 +02:00 · 2023-04-20 14:42:27 +02:00 · fc8786b149
parent 920b830bd1
commit fc8786b149
10 changed files with 167 additions and 170 deletions
--- a/.github/workflows/e2e-tests.yml
+++ b/.github/workflows/e2e-tests.yml
@ -33,7 +33,7 @@ jobs:
            3.0.x
      - name: Verify dotnet
        shell: pwsh
-        run: __tests__/verify-dotnet.ps1 2.2.402 3.1.404 '3.0'
+        run: __tests__/verify-dotnet.ps1 -Patterns "^2.2.402$", "^3.1.404$", "^3.0"

  test-setup-full-version:
    runs-on: ${{ matrix.operating-system }}
@ -60,13 +60,9 @@ jobs:
          source-url: https://api.nuget.org/v3/index.json
        env:
          NUGET_AUTH_TOKEN: NOTATOKEN
-      - name: Verify nuget config file
-        shell: pwsh
-        run: |
-          if (-Not (Test-Path "../nuget.config")) { throw "nuget file not generated correctly" }
      - name: Verify dotnet
        shell: pwsh
-        run: __tests__/verify-dotnet.ps1 3.1.201 2.2.402
+        run: __tests__/verify-dotnet.ps1 -Patterns "^3.1.201$", "^2.2.402$" -CheckNugetConfig

  test-setup-without-patch-version:
    runs-on: ${{ matrix.operating-system }}
@ -91,7 +87,7 @@ jobs:
          dotnet-version: '2.2'
      - name: Verify dotnet
        shell: pwsh
-        run: __tests__/verify-dotnet.ps1 3.1 2.2
+        run: __tests__/verify-dotnet.ps1 -Patterns "^3.1", "^2.2"

  test-setup-prerelease-version:
    runs-on: ${{ matrix.operating-system }}
@ -105,17 +101,13 @@ jobs:
      - name: Clear toolcache
        shell: pwsh
        run: __tests__/clear-toolcache.ps1 ${{ runner.os }}
-      - name: Setup dotnet '2.2'
-        uses: ./
-        with:
-          dotnet-version: '2.2'
      - name: Setup dotnet '3.1.100-preview1-014459'
        uses: ./
        with:
          dotnet-version: '3.1.100-preview1-014459'
      - name: Verify dotnet
        shell: pwsh
-        run: __tests__/verify-dotnet.ps1 3.1.100-preview1-014459
+        run: __tests__/verify-dotnet.ps1 -Patterns "3.1.100-preview1-014459"

  test-setup-latest-patch-version:
    runs-on: ${{ matrix.operating-system }}
@ -139,7 +131,7 @@ jobs:
          dotnet-version: 2.2.X
      - name: Verify dotnet
        shell: pwsh
-        run: __tests__/verify-dotnet.ps1 '2.2' '3.1'
+        run: __tests__/verify-dotnet.ps1 -Patterns "^2.2", "^3.1"

  test-setup-with-wildcard:
    runs-on: ${{ matrix.operating-system }}
@ -163,7 +155,7 @@ jobs:
          dotnet-version: 2.2.*
      - name: Verify dotnet
        shell: pwsh
-        run: __tests__/verify-dotnet.ps1 3.1 2.2
+        run: __tests__/verify-dotnet.ps1 -Patterns "^3.1", "^2.2"

  test-setup-global-json-specified-and-version:
    runs-on: ${{ matrix.operating-system }}
@ -181,7 +173,7 @@ jobs:
        shell: bash
        run: |
          mkdir subdirectory
-          echo '{"sdk":{"version": "2.2","rollForward": "latestFeature"}}' > ./subdirectory/global.json
+          echo '{"sdk":{"version": "2.2.207","rollForward": "latestFeature"}}' > ./subdirectory/global.json
      - name: Setup dotnet
        uses: ./
        with:
@ -189,7 +181,7 @@ jobs:
          global-json-file: ./subdirectory/global.json
      - name: Verify dotnet
        shell: pwsh
-        run: __tests__/verify-dotnet.ps1 2.2 3.1
+        run: __tests__/verify-dotnet.ps1 -Patterns "^2.2", "^3.1"

  test-setup-with-dotnet-quality:
    runs-on: ${{ matrix.operating-system }}
@ -209,12 +201,9 @@ jobs:
        with:
          dotnet-version: '7.0'
          dotnet-quality: 'preview'
-      - name: Verify preview version
+      - name: Verify dotnet
        shell: pwsh
-        run: |
-          $version = & dotnet --version
-          Write-Host "Installed version: $version"
-          if (-not ($version.Contains("preview") -or $version.Contains("rc"))) { throw "Unexpected version" }
+        run: __tests__/verify-dotnet.ps1 -Patterns "^7\.0\.\d+-"

  test-dotnet-version-output-during-single-version-installation:
    runs-on: ${{ matrix.operating-system }}
@ -273,7 +262,7 @@ jobs:
  test-proxy:
    runs-on: ubuntu-latest
    container:
-      image: mcr.microsoft.com/dotnet/core/runtime-deps:3.0-bionic
+      image: ubuntu:latest
      options: --dns 127.0.0.1
    services:
      squid-proxy:
@ -286,21 +275,29 @@ jobs:
    steps:
      - name: Checkout
        uses: actions/checkout@v3
-      - name: Clear tool cache
-        run: rm -rf "/usr/share/dotnet"
-      - name: Install curl
+      - name: Install Powershell
        run: |
-          apt update
-          apt -y install curl
-      - name: Setup dotnet 3.1.201
+          apt-get update
+          apt-get install -y wget apt-transport-https software-properties-common
+          wget -q "https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/packages-microsoft-prod.deb"
+          dpkg -i packages-microsoft-prod.deb
+          rm packages-microsoft-prod.deb
+          apt-get update
+          apt-get install -y powershell
+      - name: Clear toolcache
+        shell: pwsh
+        run: __tests__/clear-toolcache.ps1 ${{ runner.os }}
+      - name: Setup dotnet 6.0
        uses: ./
        with:
-          dotnet-version: 3.1.201
+          dotnet-version: 6.0
          source-url: https://api.nuget.org/v3/index.json
        env:
          NUGET_AUTH_TOKEN: NOTATOKEN
      - name: Verify dotnet
-        run: __tests__/verify-dotnet.sh 3.1.201
+        shell: pwsh
+        run: |
+          __tests__/verify-dotnet.ps1 -Patterns "^6.0" -CheckNugetConfig

  test-bypass-proxy:
    runs-on: ubuntu-latest
@ -310,8 +307,9 @@ jobs:
    steps:
      - name: Checkout
        uses: actions/checkout@v3
-      - name: Clear tool cache
-        run: rm -rf "/usr/share/dotnet"
+      - name: Clear toolcache
+        shell: pwsh
+        run: __tests__/clear-toolcache.ps1 ${{ runner.os }}
      - name: Setup dotnet 3.1.201
        uses: ./
        with:
@ -320,4 +318,5 @@ jobs:
        env:
          NUGET_AUTH_TOKEN: NOTATOKEN
      - name: Verify dotnet
-        run: __tests__/verify-dotnet.sh 3.1.201
+        shell: pwsh
+        run: __tests__/verify-dotnet.ps1 -Patterns "^3.1.201$" -CheckNugetConfig
--- a/.github/workflows/test-dotnet.yml
+++ b/.github/workflows/test-dotnet.yml
@ -18,7 +18,7 @@ jobs:
      fail-fast: false
      matrix:
        operating-system: [ubuntu-latest, windows-latest, macOS-latest]
-        dotnet-version: ['2.1', '2.2', '3.0', '3.1', '5.0']
+        dotnet-version: ['2.1', '2.2', '3.0', '3.1', '5.0', '6.0', '7.0', '8.0']
    steps:
      - name: Checkout
        uses: actions/checkout@v3
@ -29,9 +29,7 @@ jobs:
        uses: ./
        with:
          dotnet-version: ${{ matrix.dotnet-version }}
-      - name: Check installed version
+      - name: Verify installed version
        shell: pwsh
        run: |
-          $version = & dotnet --version
-          Write-Host "Installed version: $version"
-          if (-not $version.StartsWith("${{ matrix.dotnet-version }}")) { throw "Unexpected version" }
+          __tests__/verify-dotnet.ps1 -Patterns "^${{ matrix.dotnet-version }}"
--- a/.gitignore
+++ b/.gitignore
@ -3,8 +3,8 @@ global.json
 lib/
 node_modules/
 __tests__/runner/*
-__tests__/sample-csproj/bin/
-__tests__/sample-csproj/obj/
+__tests__/e2e-test-csproj/bin/
+__tests__/e2e-test-csproj/obj/

 # Rest of the file pulled from https://github.com/github/gitignore/blob/master/Node.gitignore
 # Logs
--- a/tests/e2e-test-csproj/Test.cs
+++ b/tests/e2e-test-csproj/Test.cs
@ -0,0 +1,18 @@
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using System;
+
+namespace test_csproj
+{
+    [TestClass]
+    public class Test
+    {
+        [TestMethod]
+        public void TestMethod()
+        {   
+            Console.WriteLine("TestMethod");
+            int calculatedResult = 1000 / 25;
+            int expectedResult = 40;
+            Assert.AreEqual(calculatedResult, expectedResult);
+        }
+    }
+}
--- a/tests/e2e-test-csproj/test.csproj
+++ b/tests/e2e-test-csproj/test.csproj
@ -0,0 +1,15 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>$(TEST_TARGET_FRAMEWORK)</TargetFramework>
+    <IsPackable>false</IsPackable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <!-- These packages will be downloaded over the network for testing proxy settings -->
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.5.0-preview-20170810-02" />
+    <PackageReference Include="MSTest.TestAdapter" Version="1.1.18" />
+    <PackageReference Include="MSTest.TestFramework" Version="1.1.18" />
+  </ItemGroup>
+
+</Project>
--- a/tests/sample-csproj/Program.cs
+++ b/tests/sample-csproj/Program.cs
@ -1,15 +0,0 @@
-using Microsoft.VisualStudio.TestTools.UnitTesting;
-using System;
-
-namespace sample_csproj
-{
-    [TestClass]
-    public class Program
-    {
-        [TestMethod]
-        public void TestMethod1()
-        {
-            Console.WriteLine("Hello, World!");
-        }
-    }
-}
--- a/tests/sample-csproj/sample.csproj
+++ b/tests/sample-csproj/sample.csproj
@ -1,18 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <TargetFrameworks>netcoreapp3.1;netcoreapp3.0;netcoreapp2.2</TargetFrameworks>
-    <RootNamespace>sample_csproj</RootNamespace>
-
-    <IsPackable>false</IsPackable>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <!-- These packages will be downloaded over the network for testing proxy settings -->
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0" />
-    <PackageReference Include="MSTest.TestAdapter" Version="2.1.0" />
-    <PackageReference Include="MSTest.TestFramework" Version="2.1.0" />
-    <PackageReference Include="coverlet.collector" Version="1.2.0" />
-  </ItemGroup>
-
-</Project>
--- a/tests/verify-dotnet.ps1
+++ b/tests/verify-dotnet.ps1
@ -1,73 +1,117 @@
-if (!$args[0])
-{
-  throw "Must supply dotnet version argument"
+<#
+  .DESCRIPTION
+  Verifies that installed on the machine .NET SDK versions match the input patterns.
+  Optionally checks that the nuget.config file is generated correctly.
+
+  .PARAMETER Patterns
+  Specifies the regular expression patterns that should be matched with the installed
+  on the machine .NET SDK versions. The number of patterns should be equal to the number
+  of installed .NET versions. 
+
+  .PARAMETER CheckNugetConfig
+  Switches the check for the existence of the nuget.config file.
+
+  .EXAMPLE
+  PS> .\verify-dotnet.ps1 -Paterns "^3.1.200$", "^6.0" -CheckNugetConfig
+#>
+
+param(
+  [ValidateNotNullOrEmpty()]
+  [Parameter(Mandatory=$true)]
+  [string[]]$Patterns,
+  [switch]$CheckNugetConfig
+)
+
+$PatternsList = [System.Collections.ArrayList]($Patterns)
+
+if ($CheckNugetConfig.IsPresent -and !(Test-Path "../nuget.config")) {
+  throw "The nuget.config file is not generated correctly."
 }

+Write-Host "These patterns were supplied to the script: $($PatternsList -join ', ')."
 $dotnet = Get-Command dotnet | Select-Object -First 1 | ForEach-Object { $_.Path }
-Write-Host "Found '$dotnet'"
+Write-Host "Found: '$dotnet'"

-if($args.count -eq 1)
+# SDKs are listed on multiple lines with the path afterwards in square brackets
+$Versions = & $dotnet --list-sdks | ForEach-Object { $_.SubString(0, $_.IndexOf('[')).Trim() }
+Write-Host "Found installed versions: $($Versions -join ', ')."
+$InstalledVersionCount = $Versions.Count
+
+foreach($version in $Versions)
 {
-  $version = & $dotnet --version | Out-String | ForEach-Object { $_.Trim() }
-  Write-Host "Version $version"
-  if (-not ($version.StartsWith($args[0].ToString())))
+  foreach($pattern in $PatternsList) 
  {
-    Write-Host "PATH='$env:PATH'"
-    throw "Unexpected version"
-  }
-}
-
-if ($args[1])
-{
-  # SDKs are listed on multiple lines with the path afterwards in square brackets
-  $versions = & $dotnet --list-sdks | ForEach-Object { $_.SubString(0, $_.IndexOf('[')).Trim() }
-  Write-Host "Installed versions: $versions"
-  $InstalledVersionCount = 0
-  foreach($arg in $args){
-    foreach ($version in $versions)
+    if ($version -match $pattern)
    { 
-      if ($version.StartsWith($arg.ToString())) 
-      {
-        $InstalledVersionCount++
-      }
+      $PatternsList.Remove($pattern)
+      $InstalledVersionCount--
+      break
    }
-   }
-  if ( $InstalledVersionCount -ne $args.Count)
-  {
-    Write-Host "PATH='$env:PATH'"
-    throw "Unexpected version"
  }
 }

-Write-Host "Building sample csproj"
-& $dotnet build __tests__/sample-csproj/ --no-cache
-if ($LASTEXITCODE -ne 0)
+if ( $InstalledVersionCount -ne 0)
 {
-  throw "Unexpected exit code $LASTEXITCODE"
+  throw "An unexpected version of Dotnet is found on the machine, please check the correctness of the -Patterns input."
 }

-Write-Host "Testing compiled app"
-$sample_output = "$(dotnet test __tests__/sample-csproj/ --no-build)"
-Write-Host "Sample output: $sample_output"
-# For Side-by-Side installs we want to run the tests twice, for a single install the tests will run once
-if ($args[1])
+$workingDir = Get-Location
+$testProjectDir = "./__tests__/e2e-test-csproj"
+Write-Host "Changing directory to the $testProjectDir"
+Set-Location $testProjectDir
+
+$targetFrameworkVersionMap = @{
+  "1.0" = "netcoreapp1.0";
+  "1.1" = "netcoreapp1.1";
+  "2.0" = "netcoreapp2.0";
+  "2.1" = "netcoreapp2.1";
+  "2.2" = "netcoreapp2.2";
+  "3.0" = "netcoreapp3.0";
+  "3.1" = "netcoreapp3.1";
+  "5.0" = "net5.0";
+  "6.0" = "net6.0";
+  "7.0" = "net7.0";
+  "8.0" = "net8.0";
+ }
+
+foreach ($version in $Versions)
 {
-  if ($sample_output -notlike "*Test Run Successful.*Test Run Successful.*")
+  # Creating temporary global.json file inside e2e-test-csproj dir and setting exact version of .NET inside allows to override default behavior of .NET and run build and tests on that exact version. 
+  Write-Host "Creating temporary global.json file for $version .NET version."
+  & $dotnet new globaljson --sdk-version $version --force | Out-Null
+  if (!(Test-Path "./global.json"))
  {
-    throw "Unexpected output"
+    throw "An error occured while creating the global.json file. Exit code: $LASTEXITCODE"
  }
-}
-if ($args[2])
-{
-  if ($sample_output -notlike "*Test Run Successful.*Test Run Successful.*Test Run Successful.*")
+  Write-Host "The global.json file for the version $version is created. Currently used .NET version is: $(& $dotnet --version)."
+
+  # Environment variable TEST_TARGET_FRAMEWORK is used inside the test.csproj file to target required framework version
+  $version -match "^(?<key>\d+\.\d+)" | Out-Null
+  if (!($targetFrameworkVersionMap.ContainsKey($Matches.key)))
  {
-    throw "Unexpected output"
+    throw "The map with the framework targets doesn't contain a target name for the version $version."
  }
-}
-else
-{
-  if ($sample_output -notlike "*Test Run Successful.*")
+  Write-Host "Setting the TEST_TARGET_FRAMEWORK environment variable to $($targetFrameworkVersionMap[$Matches.key])"
+  [Environment]::SetEnvironmentVariable('TEST_TARGET_FRAMEWORK', $($targetFrameworkVersionMap[$Matches.key]))
+
+  Write-Host "Building test C# project with $version .NET version."
+  & $dotnet build --no-cache
+  if ($LASTEXITCODE -ne 0)
  {
-    throw "Unexpected output"
+    throw "Building process is not successful, exit code: $LASTEXITCODE"
  }
+
+  Write-Host "Testing compiled C# project with $version .NET version."
+  & $dotnet test --no-build
+  if ($LASTEXITCODE -ne 0)
+  {
+    throw "Testing process is not successful, exit code: $LASTEXITCODE"
+  }
+
+  Write-Host "Tests are completed successfully!"
+
+  Write-Host "Removing temporary global.json file."
+  Remove-Item ./global.json
 }
+
+Set-Location $workingDir
--- a/tests/verify-dotnet.sh
+++ b/tests/verify-dotnet.sh
@ -1,44 +0,0 @@
-if [ -z "$1" ]; then
-  echo "Must supply dotnet version argument"
-  exit 1
-fi
-
-if [ ! -f "../nuget.config" ]; then
-  echo "nuget file not generated correctly"
-  exit 1
-fi
-
-dotnet_version="$(dotnet --version)"
-echo "Found dotnet version '$dotnet_version'"
-if [ -z "$(echo $dotnet_version | grep $1)" ]; then
-  echo "Unexpected version"
-  exit 1
-fi
-
-if [ -n "$2" ]; then
-  dotnet_version="$(dotnet --list-sdks)"
-  echo "Found dotnet version '$dotnet_version'"
-  if [ -z "$(echo $dotnet_version | grep $2)" ]; then
-    echo "Unexpected version"
-    exit 1
-  fi
-fi
-
-echo "Building sample csproj"
-dotnet build __tests__/sample-csproj/ --no-cache || exit 1
-
-echo "Testing compiled app"
-sample_output=$(dotnet test __tests__/sample-csproj/ --no-build)
-echo "Sample output: $sample_output"
-# For Side-by-Side installs we want to run the tests twice, for a single install the tests will run once
-if [ -n "$2" ]; then
-  if [ -z "$(echo $sample_output | grep "Test Run Successful.*Test Run Successful.")" ]; then
-    echo "Unexpected output"
-    exit 1
-  fi
-else
-  if [ -z "$(echo $sample_output | grep "Test Run Successful.")" ]; then
-    echo "Unexpected output"
-    exit 1
-  fi
-fi
--- a/docs/contributors.md
+++ b/docs/contributors.md
@ -67,7 +67,7 @@ Pull requests are the easiest way to contribute changes to git repos at GitHub.
 **Learn more about how to implement tests:**

 Adding or changing tests is an integral part of making a change to the code. 
-Unit tests are in the `__tests__` folder, and end-to-end tests are in the `workflows` folder, particularly in the [workflow.yml](https://github.com/actions/setup-dotnet/blob/main/.github/workflows/workflow.yml).
+Unit tests are in the `__tests__` folder, and end-to-end tests are in the `workflows` folder, particularly in the [e2e-tests.yml](https://github.com/actions/setup-dotnet/blob/main/.github/workflows/e2e-tests.yml).

 - The contributor can add various types of tests (like unit tests or end-to-end tests), which, in his opinion, will be necessary and sufficient for testing new or changed functionality
 - Tests should cover a successful execution, as well as some edge cases and possible errors