diff --git a/eng/pipelines/common/templates/jobs/ci-run-tests-job.yml b/eng/pipelines/common/templates/jobs/ci-run-tests-job.yml index 02cc6d58de..5ac34e017f 100644 --- a/eng/pipelines/common/templates/jobs/ci-run-tests-job.yml +++ b/eng/pipelines/common/templates/jobs/ci-run-tests-job.yml @@ -45,7 +45,11 @@ parameters: - name: operatingSystem type: string - default: '' + values: + - Linux + - Mac + - Windows + default: Windows - name: buildConfiguration type: string @@ -134,22 +138,6 @@ jobs: - ${{ if ne(parameters.prebuildSteps, '') }}: - ${{ parameters.prebuildSteps }} # extra steps to run before the build like downloading sni and the required configuration - - powershell: | - $guid = [guid]::NewGuid().ToString() - Write-Host "##vso[task.setvariable variable=password;issecret=true]$guid" - displayName: 'Generate Password' - - - powershell: | - Write-Host "Password: $(password)" # prints *** to logs when set - - $variableValue = "$(password)" - if (-not [string]::IsNullOrEmpty($variableValue)) { - Write-Output "The password exists with a value." - } else { - Write-Output "The password does not exist or is empty." - } - displayName: 'Verify Password' - - ${{ if eq(parameters.referenceType, 'Project') }}: - template: ../steps/ci-project-build-step.yml@self parameters: diff --git a/eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml b/eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml index 7568f01608..bb72dd9c62 100644 --- a/eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml +++ b/eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml @@ -4,15 +4,14 @@ # See the LICENSE file in the project root for more information. # ################################################################################# -# This step configures an existing SQL Server running on the local Linux host. -# For example, our 1ES Hosted Pool has images like ADO-UB20-SQL22 that come with -# SQL Server 2022 pre-installed and running. +# This step configures an existing SQL Server running on the local Linux host. For example, our 1ES +# Hosted Pool has images like ADO-UB20-SQL22 that come with SQL Server 2022 pre-installed and +# running. +# +# The SA password is set to the value of the $(Password) variable defined in the ADO Library "ADO +# Test Configuration properties", brought in by common/templates/libraries/ci-build-variables.yml. parameters: - - name: password - type: string - default: $(password) - - name: condition type: string default: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) @@ -21,20 +20,20 @@ steps: # Linux only steps - bash: | sudo systemctl stop mssql-server - + # Password for the SA user (required) - MSSQL_SA_PW=${{parameters.password }} + MSSQL_SA_PW=$(Password) # Product ID of the version of SQL server you're installing # Must be evaluation, developer, express, web, standard, enterprise, or your 25 digit product key MSSQL_PID='enterprise' - + echo Running mssql-conf setup... sudo MSSQL_SA_PASSWORD=$MSSQL_SA_PW\ MSSQL_PID=$MSSQL_PID \ /opt/mssql/bin/mssql-conf -n setup accept-eula - + # Connect to server and get the version: counter=1 errstatus=1 @@ -50,7 +49,7 @@ steps: errstatus=$? ((counter++)) done - + # Display error if connection failed: if [ $errstatus = 1 ] then diff --git a/eng/pipelines/common/templates/steps/configure-sql-server-macos-step.yml b/eng/pipelines/common/templates/steps/configure-sql-server-macos-step.yml index cbdfef83ec..03502c409b 100644 --- a/eng/pipelines/common/templates/steps/configure-sql-server-macos-step.yml +++ b/eng/pipelines/common/templates/steps/configure-sql-server-macos-step.yml @@ -4,14 +4,12 @@ # See the LICENSE file in the project root for more information. # ################################################################################# -# This step installs the latest SQL Server 2022 onto the macOS host and -# configures it for use. +# This step installs the latest SQL Server 2022 onto the macOS host and configures it for use. +# +# The SA password is set to the value of the $(Password) variable defined in the ADO Library "ADO +# Test Configuration properties", brought in by common/templates/libraries/ci-build-variables.yml. parameters: - - name: password - type: string - default: $(password) - - name: condition type: string default: and(succeeded(), eq(variables['Agent.OS'], 'Darwin')) @@ -19,11 +17,18 @@ parameters: steps: # macOS only steps - bash: | - # The "user" pipeline variable conflicts with homebrew, causing errors during install. Set it back to the pipeline user. + # The "user" pipeline variable conflicts with homebrew, causing errors during install. Set it + # back to the pipeline user. USER=`whoami` + SQLCMD_ERRORS=$(Agent.TempDirectory)/sqlcmd_err.log - echo $SQLCMD_ERRORS + echo "Errors will be written to: $SQLCMD_ERRORS" + # Configure the prompt to show the current timestamp so we can see how long each command takes. + export PS4='+ [$(date "+%Y-%m-%d %H:%M:%S")] ' + set -x + + # Install Docker and SQLCMD tools. brew install colima brew install --cask docker brew tap microsoft/mssql-release https://github.com/Microsoft/homebrew-mssql-release @@ -32,12 +37,12 @@ steps: colima start --arch x86_64 docker --version docker pull mcr.microsoft.com/mssql/server:2022-latest - + # Password for the SA user (required) - MSSQL_SA_PW=${{ parameters.password }} + MSSQL_SA_PW=$(Password) docker run -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=$MSSQL_SA_PW" -p 1433:1433 -p 1434:1434 --name sql1 --hostname sql1 -d mcr.microsoft.com/mssql/server:2022-latest - + sleep 5 docker ps -a @@ -50,10 +55,10 @@ steps: # Wait 3 seconds between attempts. delay=3 - + # Try up to 40 times (2 minutes) to connect. maxAttempts=40 - + # Attempt counter. attempt=1 @@ -64,18 +69,18 @@ steps: do echo "Waiting for SQL Server to start (attempt #$attempt of $maxAttempts)..." - + sqlcmd -S 127.0.0.1 -No -U sa -P $MSSQL_SA_PW -Q "SELECT @@VERSION" >> $SQLCMD_ERRORS 2>&1 - + # If the command was successful, then the SQL Server is ready. if [ $? -eq 0 ]; then ready=1 break fi - + # Increment the attempt counter. ((attempt++)) - + # Wait before trying again. sleep $delay diff --git a/eng/pipelines/common/templates/steps/configure-sql-server-step.yml b/eng/pipelines/common/templates/steps/configure-sql-server-step.yml index 884a2d3a00..822070e5ca 100644 --- a/eng/pipelines/common/templates/steps/configure-sql-server-step.yml +++ b/eng/pipelines/common/templates/steps/configure-sql-server-step.yml @@ -54,19 +54,19 @@ parameters: default: $(LocalDbSharedInstanceName) # Common parameters - - name: password - type: string - default: $(password) - - name: netcoreVersionTestUtils type: string - name: databaseName type: string default: Northwind - + - name: operatingSystem type: string + values: + - Linux + - Mac + - Windows steps: - ${{ if eq(parameters.operatingSystem, 'Windows') }}: @@ -85,19 +85,14 @@ steps: enableLocalDB: ${{parameters.enableLocalDB}} localDbAppName: ${{parameters.localDbAppName}} localDbSharedInstanceName: ${{parameters.localDbSharedInstanceName}} - password: ${{parameters.password}} - ${{ elseif eq(parameters.operatingSystem, 'Linux') }}: # Linux only steps - template: /eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml@self - parameters: - password: ${{parameters.password}} - ${{ elseif eq(parameters.operatingSystem, 'Mac') }}: # macOS only steps - template: /eng/pipelines/common/templates/steps/configure-sql-server-macos-step.yml@self - parameters: - password: ${{parameters.password}} # Common steps - task: DotNetCoreCLI@2 diff --git a/eng/pipelines/common/templates/steps/configure-sql-server-win-step.yml b/eng/pipelines/common/templates/steps/configure-sql-server-win-step.yml index 6586450e2e..a5f7cef966 100644 --- a/eng/pipelines/common/templates/steps/configure-sql-server-win-step.yml +++ b/eng/pipelines/common/templates/steps/configure-sql-server-win-step.yml @@ -4,9 +4,12 @@ # See the LICENSE file in the project root for more information. # ################################################################################# -# This step configures an existing SQL Server running on the local Windows host. -# For example, our 1ES Hosted Pool has images like ADO-MMS22-SQL22 that come -# with SQL Server 2022 pre-installed and running. +# This step configures an existing SQL Server running on the local Windows host. For example, our +# 1ES Hosted Pool has images like ADO-MMS22-SQL22 that come with SQL Server 2022 pre-installed and +# running. +# +# The SA password is set to the value of the $(Password) variable defined in the ADO Library "ADO +# Test Configuration properties", brought in by common/templates/libraries/ci-build-variables.yml. parameters: # Windows only parameters @@ -59,10 +62,6 @@ parameters: default: $(LocalDbSharedInstanceName) # Common parameters - - name: password - type: string - default: $(password) - - name: condition type: string default: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) @@ -76,19 +75,19 @@ steps: Import-Module "sqlps" $smo = 'Microsoft.SqlServer.Management.Smo.' $wmi = new-object ($smo + 'Wmi.ManagedComputer'). - # List the object properties, including the instance names. - $Wmi - - # Enable the TCP protocol on the default instance. + # List the object properties, including the instance names. + $Wmi + + # Enable the TCP protocol on the default instance. $Tcp = $wmi.GetSmoObject("ManagedComputer[@Name='$env:COMPUTERNAME']/ ServerInstance[@Name='${{parameters.instanceName }}']/ServerProtocol[@Name='Tcp']") $Tcp.IsEnabled = $true $Tcp.Alter() - + # Enable the NP protocol on the default instance. $Np = $wmi.GetSmoObject("ManagedComputer[@Name='$env:COMPUTERNAME']/ ServerInstance[@Name='${{parameters.instanceName }}']/ServerProtocol[@Name='Np']") $Np.IsEnabled = $true $Np.Alter() - + $Tcp } catch @@ -96,7 +95,7 @@ steps: $error[0] | format-list -force throw } - + New-NetFirewallRule -DisplayName "SQL TCP Ports" -Direction Inbound –Protocol TCP –LocalPort 1433 -Action allow $sqlSrvPath = (Get-WmiObject win32_service | ?{$_.DisplayName -eq 'SQL Server (${{parameters.instanceName }})'} | select @{Name="Path"; Expression={$_.PathName.split('"')[1]}}).Path New-NetFirewallRule -DisplayName "sqlservr.exe" -Program "$sqlSrvPath" @@ -105,14 +104,14 @@ steps: retryCountOnTaskFailure: 2 - powershell: | - $password = "${{parameters.password }}" + $password = "$(Password)" $machineName = $env:COMPUTERNAME - + if ("${{parameters.instanceName }}" -ne "MSSQLSERVER"){ $machineName += "\${{parameters.instanceName }}" } - + Write-Host $machineName Import-Module "sqlps" $tries = 0 @@ -141,7 +140,7 @@ steps: condition: ${{parameters.condition }} env: SQL_USER: ${{parameters.user }} - SQL_PASSWD: ${{parameters.password }} + SQL_PASSWD: $(Password) - ${{ if ne(parameters.SQLRootPath, '') }}: - powershell: | @@ -149,13 +148,13 @@ steps: $instance = "${{parameters.instanceName }}" $wmi = Get-WmiObject -Namespace "${{parameters.SQLRootPath }}" -Class FilestreamSettings | where {$_.InstanceName -eq $instance} $wmi.EnableFilestream(3, $instance) - + $machineName = $env:COMPUTERNAME - + if ("${{parameters.instanceName }}" -ne "MSSQLSERVER"){ $machineName += "\${{parameters.instanceName }}" } - + #Change the access level for FileStream for SQLServer Set-ExecutionPolicy Unrestricted Import-Module "sqlps" @@ -167,7 +166,7 @@ steps: condition: ${{parameters.condition }} env: SQL_USER: ${{parameters.user }} - SQL_PASSWD: ${{parameters.password }} + SQL_PASSWD: $(Password) - ${{ if ne(parameters.FileStreamDirectory, '') }}: - powershell: | @@ -180,17 +179,17 @@ steps: - powershell: | $SQLServerName = ("{0}" -f [System.Net.Dns]::GetHostByName($env:computerName).HostName) Write-Host FQDN is: $SQLServerName - + if ((Test-Path -Path ${{parameters.x64AliasRegistryPath }}) -ne $true) { New-Item ${{parameters.x64AliasRegistryPath }} } - + if ((Test-Path -Path ${{parameters.x86AliasRegistryPath }}) -ne $true) { New-Item ${{parameters.x86AliasRegistryPath }} } - + $TCPAliasName = "DBMSSOCN, $SQLServerName, ${{parameters.SQLAliasPort }}" - + New-ItemProperty -Path ${{parameters.x86AliasRegistryPath }} -Name ${{parameters.SQLAliasName }} -PropertyType string -Value $TCPAliasName New-ItemProperty -Path ${{parameters.x64AliasRegistryPath }} -Name ${{parameters.SQLAliasName }} -PropertyType string -Value $TCPAliasName displayName: 'Setup SQL Alias [Win]' @@ -211,8 +210,8 @@ steps: "localmachine" ) - $store.open("MaxAllowed") - $store.add($certificate) + $store.open("MaxAllowed") + $store.add($certificate) $store.close() # Get SQL Server instances and add the Certificate @@ -221,7 +220,7 @@ steps: $instance | ForEach-Object { $_.PSObject.Properties | Where-Object { $_.Name -notmatch '^PS.*' } | ForEach-Object { Set-ItemProperty "HKLM:\SOFTWARE\Microsoft\Microsoft SQL Server\$($_.Value)\MSSQLServer\SuperSocketNetLib" -Name Certificate -Value $certificate.Thumbprint.ToLower() - + # Grant read access to Private Key for SQL Service Account if ($($_.Name) -eq "MSSQLSERVER") { icacls $machineKeyPath /grant "NT Service\MSSQLSERVER:R" @@ -239,31 +238,31 @@ steps: # -Force takes care of any dependent services, like SQL Agent. # Note: if the instance is named, replace MSSQLSERVER with MSSQL$ followed by # the name of the instance (e.g. MSSQL$MYINSTANCE) - + $serviceName = "${{parameters.instanceName }}" $InstancePrefix = 'MSSQL$' - + if ( "${{parameters.instanceName }}" -ne "MSSQLSERVER" ) { $serviceName = $InstancePrefix+"${{parameters.instanceName }}" } - + Restart-Service -Name "$serviceName" -Force Restart-Service -Name MSSQLSERVER* -Force - + displayName: 'Restart SQL Server [Win]' condition: ${{parameters.condition }} - powershell: | $arrService = Get-Service -Name "SQLBrowser" - $arrService - + $arrService + if ($arrService.Status -eq 'Stopped') { Write-Host 'Attempt to run the service ...' # updating the startup type to make sure it's not disabled - Set-Service -StartupType Automatic $arrService.Name + Set-Service -StartupType Automatic $arrService.Name $arrService.Start() - + $arrService.WaitForStatus('Running', '00:00:30') if ($arrService.Status -eq 'Running') { $arrService @@ -277,14 +276,14 @@ steps: - ${{ if parameters.enableLocalDB }}: - powershell: | #script to enable local db - + SqlLocalDB info #SqlLocalDB create ${{parameters.localDbAppName }} SqlLocalDB info ${{parameters.localDbAppName }} SqlLocalDB share ${{parameters.localDbAppName }} ${{parameters.LocalDbSharedInstanceName }} SqlLocalDB start ${{parameters.localDbAppName }} SqlLocalDB info ${{parameters.localDbAppName }} - + sqlcmd -S "(localdb)\.\${{parameters.LocalDbSharedInstanceName }}" -q "SELECT @@VERSION" displayName: 'Enable LocalDB [Win]' condition: ${{parameters.condition }} diff --git a/eng/pipelines/common/templates/steps/publish-test-results-step.yml b/eng/pipelines/common/templates/steps/publish-test-results-step.yml index 6eefa8ecbe..fed2f7209e 100644 --- a/eng/pipelines/common/templates/steps/publish-test-results-step.yml +++ b/eng/pipelines/common/templates/steps/publish-test-results-step.yml @@ -20,9 +20,13 @@ parameters: values: - Debug - Release - + - name: operatingSystem type: string + values: + - Linux + - Mac + - Windows steps: - task: PublishTestResults@2 @@ -45,7 +49,7 @@ steps: - powershell: | cd TestResults $TF="${{parameters.targetFramework }}" - Get-ChildItem -Filter “*.coverage” -Recurse | Rename-Item -NewName {"$TF" + $_.name} + Get-ChildItem -Filter “*.coverage” -Recurse | Rename-Item -NewName {"$TF" + $_.name} displayName: 'Rename coverage files' - ${{ if eq(parameters.debug, true)}}: diff --git a/eng/pipelines/common/templates/steps/run-all-tests-step.yml b/eng/pipelines/common/templates/steps/run-all-tests-step.yml index d88065f44a..996d413757 100644 --- a/eng/pipelines/common/templates/steps/run-all-tests-step.yml +++ b/eng/pipelines/common/templates/steps/run-all-tests-step.yml @@ -18,7 +18,7 @@ parameters: - name: platform type: string default: $(Platform) - + - name: buildConfiguration type: string values: @@ -39,14 +39,18 @@ parameters: values: - x64 - x86 - + - name: dotnetx86RootPath # full path to the x86 dotnet root folder with trailing slash type: string default: '' - + - name: operatingSystem type: string - default: 'Windows' + values: + - Linux + - Mac + - Windows + default: Windows - name: retryCountOnManualTests type: number @@ -68,7 +72,7 @@ steps: msbuildArchitecture: ${{parameters.msbuildArchitecture }} platform: '${{parameters.platform }}' configuration: '${{parameters.buildConfiguration }}' - ${{ if eq(parameters.msbuildArchitecture, 'x64') }}: + ${{ if eq(parameters.msbuildArchitecture, 'x64') }}: msbuildArguments: >- -t:RunUnitTests -p:TF=${{ parameters.targetFramework }} @@ -88,7 +92,7 @@ steps: msbuildArchitecture: ${{parameters.msbuildArchitecture }} platform: '${{parameters.platform }}' configuration: '${{parameters.buildConfiguration }}' - ${{ if eq(parameters.msbuildArchitecture, 'x64') }}: + ${{ if eq(parameters.msbuildArchitecture, 'x64') }}: msbuildArguments: >- -t:RunUnitTests -p:TF=${{ parameters.targetFramework }} @@ -113,7 +117,7 @@ steps: msbuildArchitecture: ${{parameters.msbuildArchitecture }} platform: '${{parameters.platform }}' configuration: '${{parameters.buildConfiguration }}' - ${{ if eq(parameters.msbuildArchitecture, 'x64') }}: + ${{ if eq(parameters.msbuildArchitecture, 'x64') }}: msbuildArguments: >- -t:RunFunctionalTests -p:TF=${{ parameters.targetFramework }} @@ -137,7 +141,7 @@ steps: msbuildArchitecture: ${{parameters.msbuildArchitecture }} platform: '${{parameters.platform }}' configuration: '${{parameters.buildConfiguration }}' - ${{ if eq(parameters.msbuildArchitecture, 'x64') }}: + ${{ if eq(parameters.msbuildArchitecture, 'x64') }}: msbuildArguments: >- -t:RunFunctionalTests -p:TF=${{ parameters.targetFramework }} @@ -232,7 +236,7 @@ steps: -p:Configuration=${{ parameters.buildConfiguration }} verbosityRestore: Detailed verbosityPack: Detailed - + - task: DotNetCoreCLI@2 displayName: 'Run Flaky Unit Tests' condition: succeededOrFailed() diff --git a/eng/pipelines/dotnet-sqlclient-ci-package-reference-pipeline.yml b/eng/pipelines/dotnet-sqlclient-ci-package-reference-pipeline.yml index 32ecde782c..376d3fcd90 100644 --- a/eng/pipelines/dotnet-sqlclient-ci-package-reference-pipeline.yml +++ b/eng/pipelines/dotnet-sqlclient-ci-package-reference-pipeline.yml @@ -48,11 +48,11 @@ pr: none # Trigger this pipeline on commits to certain branches. trigger: - + # Don't trigger a new run until the previous one completes. This may cause # multiple commits to be batched into a single run. batch: true - + branches: include: # GitHub main branch. @@ -60,7 +60,7 @@ trigger: # ADO internal/main branch. - internal/main - + paths: include: - .azuredevops @@ -154,7 +154,7 @@ parameters: - name: testJobTimeout displayName: Test job timeout (in minutes) type: number - default: 60 + default: 90 - name: testSets displayName: Test Sets diff --git a/eng/pipelines/dotnet-sqlclient-ci-project-reference-pipeline.yml b/eng/pipelines/dotnet-sqlclient-ci-project-reference-pipeline.yml index b149b8384a..1cec6c5cbc 100644 --- a/eng/pipelines/dotnet-sqlclient-ci-project-reference-pipeline.yml +++ b/eng/pipelines/dotnet-sqlclient-ci-project-reference-pipeline.yml @@ -48,11 +48,11 @@ pr: none # Trigger this pipeline on commits to certain branches. trigger: - + # Don't trigger a new run until the previous one completes. This may cause # multiple commits to be batched into a single run. batch: true - + branches: include: # GitHub main branch. @@ -60,7 +60,7 @@ trigger: # ADO internal/main branch. - internal/main - + paths: include: - .azuredevops @@ -154,7 +154,7 @@ parameters: - name: testJobTimeout displayName: Test job timeout (in minutes) type: number - default: 60 + default: 90 - name: testSets displayName: Test Sets diff --git a/eng/pipelines/sqlclient-pr-package-ref-pipeline.yml b/eng/pipelines/sqlclient-pr-package-ref-pipeline.yml index b875aa2cfe..5b75f1f2de 100644 --- a/eng/pipelines/sqlclient-pr-package-ref-pipeline.yml +++ b/eng/pipelines/sqlclient-pr-package-ref-pipeline.yml @@ -109,7 +109,7 @@ parameters: - name: testJobTimeout displayName: Test job timeout (in minutes) type: number - default: 60 + default: 90 - name: testSets displayName: Test Sets diff --git a/eng/pipelines/sqlclient-pr-project-ref-pipeline.yml b/eng/pipelines/sqlclient-pr-project-ref-pipeline.yml index b8f13f31aa..2f4e8a2ee5 100644 --- a/eng/pipelines/sqlclient-pr-project-ref-pipeline.yml +++ b/eng/pipelines/sqlclient-pr-project-ref-pipeline.yml @@ -109,7 +109,7 @@ parameters: - name: testJobTimeout displayName: Test job timeout (in minutes) type: number - default: 60 + default: 90 - name: testSets displayName: Test Sets diff --git a/eng/pipelines/stages/build-azure-package-ci-stage.yml b/eng/pipelines/stages/build-azure-package-ci-stage.yml index b8e049caa2..658c5fa30b 100644 --- a/eng/pipelines/stages/build-azure-package-ci-stage.yml +++ b/eng/pipelines/stages/build-azure-package-ci-stage.yml @@ -107,7 +107,7 @@ parameters: # This is used when the referenceType is 'Package'. - name: mdsPackageVersion type: string - + # The reference type to use: # # Project - dependent projects are referenced directly. @@ -123,9 +123,9 @@ stages: - stage: build_azure_package_stage displayName: Build Azure Package - + dependsOn: ${{ parameters.dependsOn }} - + jobs: # ------------------------------------------------------------------------ @@ -171,9 +171,6 @@ stages: # Override the template's default step condition. We always # want this step to run. condition: true - # Use the Azure DevOps Library variable "password" for the SA - # password. - password: $(password) vmImage: ADO-UB22-SQL22 # ------------------------------------------------------------------------ @@ -225,7 +222,6 @@ stages: # These variables are from an Azure DevOps Library variable # group. fileStreamDirectory: $(FileStreamDirectory) - password: $(password) sqlRootPath: $(SQL22RootPath) # The ADO-MMS22-SQL22 image includes a local SQL Server that supports # integrated security.