From fc8786b149b2cff0bab21db5f07780e94e258ceb Mon Sep 17 00:00:00 2001 From: Ivan <98037481+IvanZosimov@users.noreply.github.com> Date: Thu, 20 Apr 2023 14:42:27 +0200 Subject: [PATCH] Update e2e tests and e2e test infrastructure (#414) --- .github/workflows/e2e-tests.yml | 65 ++++++----- .github/workflows/test-dotnet.yml | 8 +- .gitignore | 4 +- __tests__/e2e-test-csproj/Test.cs | 18 ++++ __tests__/e2e-test-csproj/test.csproj | 15 +++ __tests__/sample-csproj/Program.cs | 15 --- __tests__/sample-csproj/sample.csproj | 18 ---- __tests__/verify-dotnet.ps1 | 148 +++++++++++++++++--------- __tests__/verify-dotnet.sh | 44 -------- docs/contributors.md | 2 +- 10 files changed, 167 insertions(+), 170 deletions(-) create mode 100644 __tests__/e2e-test-csproj/Test.cs create mode 100644 __tests__/e2e-test-csproj/test.csproj delete mode 100644 __tests__/sample-csproj/Program.cs delete mode 100644 __tests__/sample-csproj/sample.csproj delete mode 100755 __tests__/verify-dotnet.sh diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 1be4173..9963bbc 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -33,7 +33,7 @@ jobs: 3.0.x - name: Verify dotnet shell: pwsh - run: __tests__/verify-dotnet.ps1 2.2.402 3.1.404 '3.0' + run: __tests__/verify-dotnet.ps1 -Patterns "^2.2.402$", "^3.1.404$", "^3.0" test-setup-full-version: runs-on: ${{ matrix.operating-system }} @@ -60,13 +60,9 @@ jobs: source-url: https://api.nuget.org/v3/index.json env: NUGET_AUTH_TOKEN: NOTATOKEN - - name: Verify nuget config file - shell: pwsh - run: | - if (-Not (Test-Path "../nuget.config")) { throw "nuget file not generated correctly" } - name: Verify dotnet shell: pwsh - run: __tests__/verify-dotnet.ps1 3.1.201 2.2.402 + run: __tests__/verify-dotnet.ps1 -Patterns "^3.1.201$", "^2.2.402$" -CheckNugetConfig test-setup-without-patch-version: runs-on: ${{ matrix.operating-system }} @@ -91,7 +87,7 @@ jobs: dotnet-version: '2.2' - name: Verify dotnet shell: pwsh - run: __tests__/verify-dotnet.ps1 3.1 2.2 + run: __tests__/verify-dotnet.ps1 -Patterns "^3.1", "^2.2" test-setup-prerelease-version: runs-on: ${{ matrix.operating-system }} @@ -105,17 +101,13 @@ jobs: - name: Clear toolcache shell: pwsh run: __tests__/clear-toolcache.ps1 ${{ runner.os }} - - name: Setup dotnet '2.2' - uses: ./ - with: - dotnet-version: '2.2' - name: Setup dotnet '3.1.100-preview1-014459' uses: ./ with: dotnet-version: '3.1.100-preview1-014459' - name: Verify dotnet shell: pwsh - run: __tests__/verify-dotnet.ps1 3.1.100-preview1-014459 + run: __tests__/verify-dotnet.ps1 -Patterns "3.1.100-preview1-014459" test-setup-latest-patch-version: runs-on: ${{ matrix.operating-system }} @@ -139,7 +131,7 @@ jobs: dotnet-version: 2.2.X - name: Verify dotnet shell: pwsh - run: __tests__/verify-dotnet.ps1 '2.2' '3.1' + run: __tests__/verify-dotnet.ps1 -Patterns "^2.2", "^3.1" test-setup-with-wildcard: runs-on: ${{ matrix.operating-system }} @@ -163,7 +155,7 @@ jobs: dotnet-version: 2.2.* - name: Verify dotnet shell: pwsh - run: __tests__/verify-dotnet.ps1 3.1 2.2 + run: __tests__/verify-dotnet.ps1 -Patterns "^3.1", "^2.2" test-setup-global-json-specified-and-version: runs-on: ${{ matrix.operating-system }} @@ -181,7 +173,7 @@ jobs: shell: bash run: | mkdir subdirectory - echo '{"sdk":{"version": "2.2","rollForward": "latestFeature"}}' > ./subdirectory/global.json + echo '{"sdk":{"version": "2.2.207","rollForward": "latestFeature"}}' > ./subdirectory/global.json - name: Setup dotnet uses: ./ with: @@ -189,7 +181,7 @@ jobs: global-json-file: ./subdirectory/global.json - name: Verify dotnet shell: pwsh - run: __tests__/verify-dotnet.ps1 2.2 3.1 + run: __tests__/verify-dotnet.ps1 -Patterns "^2.2", "^3.1" test-setup-with-dotnet-quality: runs-on: ${{ matrix.operating-system }} @@ -209,12 +201,9 @@ jobs: with: dotnet-version: '7.0' dotnet-quality: 'preview' - - name: Verify preview version + - name: Verify dotnet shell: pwsh - run: | - $version = & dotnet --version - Write-Host "Installed version: $version" - if (-not ($version.Contains("preview") -or $version.Contains("rc"))) { throw "Unexpected version" } + run: __tests__/verify-dotnet.ps1 -Patterns "^7\.0\.\d+-" test-dotnet-version-output-during-single-version-installation: runs-on: ${{ matrix.operating-system }} @@ -273,7 +262,7 @@ jobs: test-proxy: runs-on: ubuntu-latest container: - image: mcr.microsoft.com/dotnet/core/runtime-deps:3.0-bionic + image: ubuntu:latest options: --dns 127.0.0.1 services: squid-proxy: @@ -286,21 +275,29 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 - - name: Clear tool cache - run: rm -rf "/usr/share/dotnet" - - name: Install curl + - name: Install Powershell run: | - apt update - apt -y install curl - - name: Setup dotnet 3.1.201 + apt-get update + apt-get install -y wget apt-transport-https software-properties-common + wget -q "https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/packages-microsoft-prod.deb" + dpkg -i packages-microsoft-prod.deb + rm packages-microsoft-prod.deb + apt-get update + apt-get install -y powershell + - name: Clear toolcache + shell: pwsh + run: __tests__/clear-toolcache.ps1 ${{ runner.os }} + - name: Setup dotnet 6.0 uses: ./ with: - dotnet-version: 3.1.201 + dotnet-version: 6.0 source-url: https://api.nuget.org/v3/index.json env: NUGET_AUTH_TOKEN: NOTATOKEN - name: Verify dotnet - run: __tests__/verify-dotnet.sh 3.1.201 + shell: pwsh + run: | + __tests__/verify-dotnet.ps1 -Patterns "^6.0" -CheckNugetConfig test-bypass-proxy: runs-on: ubuntu-latest @@ -310,8 +307,9 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 - - name: Clear tool cache - run: rm -rf "/usr/share/dotnet" + - name: Clear toolcache + shell: pwsh + run: __tests__/clear-toolcache.ps1 ${{ runner.os }} - name: Setup dotnet 3.1.201 uses: ./ with: @@ -320,4 +318,5 @@ jobs: env: NUGET_AUTH_TOKEN: NOTATOKEN - name: Verify dotnet - run: __tests__/verify-dotnet.sh 3.1.201 + shell: pwsh + run: __tests__/verify-dotnet.ps1 -Patterns "^3.1.201$" -CheckNugetConfig diff --git a/.github/workflows/test-dotnet.yml b/.github/workflows/test-dotnet.yml index 0b37d28..03b9a29 100644 --- a/.github/workflows/test-dotnet.yml +++ b/.github/workflows/test-dotnet.yml @@ -18,7 +18,7 @@ jobs: fail-fast: false matrix: operating-system: [ubuntu-latest, windows-latest, macOS-latest] - dotnet-version: ['2.1', '2.2', '3.0', '3.1', '5.0'] + dotnet-version: ['2.1', '2.2', '3.0', '3.1', '5.0', '6.0', '7.0', '8.0'] steps: - name: Checkout uses: actions/checkout@v3 @@ -29,9 +29,7 @@ jobs: uses: ./ with: dotnet-version: ${{ matrix.dotnet-version }} - - name: Check installed version + - name: Verify installed version shell: pwsh run: | - $version = & dotnet --version - Write-Host "Installed version: $version" - if (-not $version.StartsWith("${{ matrix.dotnet-version }}")) { throw "Unexpected version" } + __tests__/verify-dotnet.ps1 -Patterns "^${{ matrix.dotnet-version }}" diff --git a/.gitignore b/.gitignore index b339e2c..1cb082a 100644 --- a/.gitignore +++ b/.gitignore @@ -3,8 +3,8 @@ global.json lib/ node_modules/ __tests__/runner/* -__tests__/sample-csproj/bin/ -__tests__/sample-csproj/obj/ +__tests__/e2e-test-csproj/bin/ +__tests__/e2e-test-csproj/obj/ # Rest of the file pulled from https://github.com/github/gitignore/blob/master/Node.gitignore # Logs diff --git a/__tests__/e2e-test-csproj/Test.cs b/__tests__/e2e-test-csproj/Test.cs new file mode 100644 index 0000000..ba9573a --- /dev/null +++ b/__tests__/e2e-test-csproj/Test.cs @@ -0,0 +1,18 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; + +namespace test_csproj +{ + [TestClass] + public class Test + { + [TestMethod] + public void TestMethod() + { + Console.WriteLine("TestMethod"); + int calculatedResult = 1000 / 25; + int expectedResult = 40; + Assert.AreEqual(calculatedResult, expectedResult); + } + } +} diff --git a/__tests__/e2e-test-csproj/test.csproj b/__tests__/e2e-test-csproj/test.csproj new file mode 100644 index 0000000..769269f --- /dev/null +++ b/__tests__/e2e-test-csproj/test.csproj @@ -0,0 +1,15 @@ + + + + $(TEST_TARGET_FRAMEWORK) + false + + + + + + + + + + diff --git a/__tests__/sample-csproj/Program.cs b/__tests__/sample-csproj/Program.cs deleted file mode 100644 index f14c939..0000000 --- a/__tests__/sample-csproj/Program.cs +++ /dev/null @@ -1,15 +0,0 @@ -using Microsoft.VisualStudio.TestTools.UnitTesting; -using System; - -namespace sample_csproj -{ - [TestClass] - public class Program - { - [TestMethod] - public void TestMethod1() - { - Console.WriteLine("Hello, World!"); - } - } -} diff --git a/__tests__/sample-csproj/sample.csproj b/__tests__/sample-csproj/sample.csproj deleted file mode 100644 index be6d7ea..0000000 --- a/__tests__/sample-csproj/sample.csproj +++ /dev/null @@ -1,18 +0,0 @@ - - - - netcoreapp3.1;netcoreapp3.0;netcoreapp2.2 - sample_csproj - - false - - - - - - - - - - - diff --git a/__tests__/verify-dotnet.ps1 b/__tests__/verify-dotnet.ps1 index 7068c87..187c715 100755 --- a/__tests__/verify-dotnet.ps1 +++ b/__tests__/verify-dotnet.ps1 @@ -1,73 +1,117 @@ -if (!$args[0]) -{ - throw "Must supply dotnet version argument" +<# + .DESCRIPTION + Verifies that installed on the machine .NET SDK versions match the input patterns. + Optionally checks that the nuget.config file is generated correctly. + + .PARAMETER Patterns + Specifies the regular expression patterns that should be matched with the installed + on the machine .NET SDK versions. The number of patterns should be equal to the number + of installed .NET versions. + + .PARAMETER CheckNugetConfig + Switches the check for the existence of the nuget.config file. + + .EXAMPLE + PS> .\verify-dotnet.ps1 -Paterns "^3.1.200$", "^6.0" -CheckNugetConfig +#> + +param( + [ValidateNotNullOrEmpty()] + [Parameter(Mandatory=$true)] + [string[]]$Patterns, + [switch]$CheckNugetConfig +) + +$PatternsList = [System.Collections.ArrayList]($Patterns) + +if ($CheckNugetConfig.IsPresent -and !(Test-Path "../nuget.config")) { + throw "The nuget.config file is not generated correctly." } +Write-Host "These patterns were supplied to the script: $($PatternsList -join ', ')." $dotnet = Get-Command dotnet | Select-Object -First 1 | ForEach-Object { $_.Path } -Write-Host "Found '$dotnet'" +Write-Host "Found: '$dotnet'" -if($args.count -eq 1) +# SDKs are listed on multiple lines with the path afterwards in square brackets +$Versions = & $dotnet --list-sdks | ForEach-Object { $_.SubString(0, $_.IndexOf('[')).Trim() } +Write-Host "Found installed versions: $($Versions -join ', ')." +$InstalledVersionCount = $Versions.Count + +foreach($version in $Versions) { - $version = & $dotnet --version | Out-String | ForEach-Object { $_.Trim() } - Write-Host "Version $version" - if (-not ($version.StartsWith($args[0].ToString()))) + foreach($pattern in $PatternsList) { - Write-Host "PATH='$env:PATH'" - throw "Unexpected version" - } -} - -if ($args[1]) -{ - # SDKs are listed on multiple lines with the path afterwards in square brackets - $versions = & $dotnet --list-sdks | ForEach-Object { $_.SubString(0, $_.IndexOf('[')).Trim() } - Write-Host "Installed versions: $versions" - $InstalledVersionCount = 0 - foreach($arg in $args){ - foreach ($version in $versions) - { - if ($version.StartsWith($arg.ToString())) - { - $InstalledVersionCount++ - } + if ($version -match $pattern) + { + $PatternsList.Remove($pattern) + $InstalledVersionCount-- + break } - } - if ( $InstalledVersionCount -ne $args.Count) - { - Write-Host "PATH='$env:PATH'" - throw "Unexpected version" } } -Write-Host "Building sample csproj" -& $dotnet build __tests__/sample-csproj/ --no-cache -if ($LASTEXITCODE -ne 0) +if ( $InstalledVersionCount -ne 0) { - throw "Unexpected exit code $LASTEXITCODE" + throw "An unexpected version of Dotnet is found on the machine, please check the correctness of the -Patterns input." } -Write-Host "Testing compiled app" -$sample_output = "$(dotnet test __tests__/sample-csproj/ --no-build)" -Write-Host "Sample output: $sample_output" -# For Side-by-Side installs we want to run the tests twice, for a single install the tests will run once -if ($args[1]) +$workingDir = Get-Location +$testProjectDir = "./__tests__/e2e-test-csproj" +Write-Host "Changing directory to the $testProjectDir" +Set-Location $testProjectDir + +$targetFrameworkVersionMap = @{ + "1.0" = "netcoreapp1.0"; + "1.1" = "netcoreapp1.1"; + "2.0" = "netcoreapp2.0"; + "2.1" = "netcoreapp2.1"; + "2.2" = "netcoreapp2.2"; + "3.0" = "netcoreapp3.0"; + "3.1" = "netcoreapp3.1"; + "5.0" = "net5.0"; + "6.0" = "net6.0"; + "7.0" = "net7.0"; + "8.0" = "net8.0"; + } + +foreach ($version in $Versions) { - if ($sample_output -notlike "*Test Run Successful.*Test Run Successful.*") + # Creating temporary global.json file inside e2e-test-csproj dir and setting exact version of .NET inside allows to override default behavior of .NET and run build and tests on that exact version. + Write-Host "Creating temporary global.json file for $version .NET version." + & $dotnet new globaljson --sdk-version $version --force | Out-Null + if (!(Test-Path "./global.json")) { - throw "Unexpected output" + throw "An error occured while creating the global.json file. Exit code: $LASTEXITCODE" } -} -if ($args[2]) -{ - if ($sample_output -notlike "*Test Run Successful.*Test Run Successful.*Test Run Successful.*") + Write-Host "The global.json file for the version $version is created. Currently used .NET version is: $(& $dotnet --version)." + + # Environment variable TEST_TARGET_FRAMEWORK is used inside the test.csproj file to target required framework version + $version -match "^(?\d+\.\d+)" | Out-Null + if (!($targetFrameworkVersionMap.ContainsKey($Matches.key))) { - throw "Unexpected output" + throw "The map with the framework targets doesn't contain a target name for the version $version." } -} -else -{ - if ($sample_output -notlike "*Test Run Successful.*") + Write-Host "Setting the TEST_TARGET_FRAMEWORK environment variable to $($targetFrameworkVersionMap[$Matches.key])" + [Environment]::SetEnvironmentVariable('TEST_TARGET_FRAMEWORK', $($targetFrameworkVersionMap[$Matches.key])) + + Write-Host "Building test C# project with $version .NET version." + & $dotnet build --no-cache + if ($LASTEXITCODE -ne 0) { - throw "Unexpected output" + throw "Building process is not successful, exit code: $LASTEXITCODE" } + + Write-Host "Testing compiled C# project with $version .NET version." + & $dotnet test --no-build + if ($LASTEXITCODE -ne 0) + { + throw "Testing process is not successful, exit code: $LASTEXITCODE" + } + + Write-Host "Tests are completed successfully!" + + Write-Host "Removing temporary global.json file." + Remove-Item ./global.json } + +Set-Location $workingDir \ No newline at end of file diff --git a/__tests__/verify-dotnet.sh b/__tests__/verify-dotnet.sh deleted file mode 100755 index 098d076..0000000 --- a/__tests__/verify-dotnet.sh +++ /dev/null @@ -1,44 +0,0 @@ -if [ -z "$1" ]; then - echo "Must supply dotnet version argument" - exit 1 -fi - -if [ ! -f "../nuget.config" ]; then - echo "nuget file not generated correctly" - exit 1 -fi - -dotnet_version="$(dotnet --version)" -echo "Found dotnet version '$dotnet_version'" -if [ -z "$(echo $dotnet_version | grep $1)" ]; then - echo "Unexpected version" - exit 1 -fi - -if [ -n "$2" ]; then - dotnet_version="$(dotnet --list-sdks)" - echo "Found dotnet version '$dotnet_version'" - if [ -z "$(echo $dotnet_version | grep $2)" ]; then - echo "Unexpected version" - exit 1 - fi -fi - -echo "Building sample csproj" -dotnet build __tests__/sample-csproj/ --no-cache || exit 1 - -echo "Testing compiled app" -sample_output=$(dotnet test __tests__/sample-csproj/ --no-build) -echo "Sample output: $sample_output" -# For Side-by-Side installs we want to run the tests twice, for a single install the tests will run once -if [ -n "$2" ]; then - if [ -z "$(echo $sample_output | grep "Test Run Successful.*Test Run Successful.")" ]; then - echo "Unexpected output" - exit 1 - fi -else - if [ -z "$(echo $sample_output | grep "Test Run Successful.")" ]; then - echo "Unexpected output" - exit 1 - fi -fi \ No newline at end of file diff --git a/docs/contributors.md b/docs/contributors.md index 0141e97..3d2a55d 100644 --- a/docs/contributors.md +++ b/docs/contributors.md @@ -67,7 +67,7 @@ Pull requests are the easiest way to contribute changes to git repos at GitHub. **Learn more about how to implement tests:** Adding or changing tests is an integral part of making a change to the code. -Unit tests are in the `__tests__` folder, and end-to-end tests are in the `workflows` folder, particularly in the [workflow.yml](https://github.com/actions/setup-dotnet/blob/main/.github/workflows/workflow.yml). +Unit tests are in the `__tests__` folder, and end-to-end tests are in the `workflows` folder, particularly in the [e2e-tests.yml](https://github.com/actions/setup-dotnet/blob/main/.github/workflows/e2e-tests.yml). - The contributor can add various types of tests (like unit tests or end-to-end tests), which, in his opinion, will be necessary and sufficient for testing new or changed functionality - Tests should cover a successful execution, as well as some edge cases and possible errors