diff --git a/azure-pipelines-PR.yml b/azure-pipelines-PR.yml index 9f06d74493..014e5fb1d4 100644 --- a/azure-pipelines-PR.yml +++ b/azure-pipelines-PR.yml @@ -720,24 +720,48 @@ stages: pool: name: $(DncEngPublicBuildPool) demands: ImageOverride -equals $(WindowsMachineQueueName) - strategy: - maxParallel: 2 - matrix: - regular: - _experimental_flag: '' - experimental_features: - _experimental_flag: '' steps: - checkout: self clean: true - script: .\Build.cmd -c Release -pack env: NativeToolsOnMachine: true - FSHARP_EXPERIMENTAL_FEATURES: $(_experimental_flag) - script: .\tests\EndToEndBuildTests\EndToEndBuildTests.cmd -c Release - env: - FSHARP_EXPERIMENTAL_FEATURES: $(_experimental_flag) displayName: End to end build tests + + # Publish artifacts for regression testing + - task: PublishPipelineArtifact@1 + displayName: Publish F# Compiler FSC Artifacts for Regression Tests + inputs: + targetPath: '$(Build.SourcesDirectory)/artifacts/bin/fsc' + artifactName: 'FSharpCompilerFscArtifacts' + publishLocation: pipeline + condition: succeeded() + + - task: PublishPipelineArtifact@1 + displayName: Publish F# Core Artifacts for Regression Tests + inputs: + targetPath: '$(Build.SourcesDirectory)/artifacts/bin/FSharp.Core' + artifactName: 'FSharpCoreArtifacts' + publishLocation: pipeline + condition: succeeded() + + - task: PublishPipelineArtifact@1 + displayName: Publish UseLocalCompiler props file for Regression Tests + inputs: + targetPath: '$(Build.SourcesDirectory)/UseLocalCompiler.Directory.Build.props' + artifactName: 'UseLocalCompilerProps' + publishLocation: pipeline + condition: succeeded() + + # F# Compiler Regression Tests using third-party libraries + - template: /eng/templates/regression-test-jobs.yml + parameters: + testMatrix: + - repo: fsprojects/FSharpPlus + commit: f614035b75922aba41ed6a36c2fc986a2171d2b8 + buildScript: build.cmd + displayName: FSharpPlus # Up-to-date - disabled due to it being flaky #- job: UpToDate_Windows diff --git a/docs/regression-testing-pipeline.md b/docs/regression-testing-pipeline.md new file mode 100644 index 0000000000..7d617fb446 --- /dev/null +++ b/docs/regression-testing-pipeline.md @@ -0,0 +1,174 @@ +# F# Compiler Regression Testing + +This document describes the F# compiler regression testing functionality implemented as a reusable Azure DevOps template in `eng/templates/regression-test-jobs.yml` and integrated into the main PR pipeline (`azure-pipelines-PR.yml`). + +## Purpose + +The regression testing helps catch F# compiler regressions by building popular third-party F# libraries with the freshly built compiler from this repository. This provides early detection of breaking changes that might affect real-world F# projects. + +## How It Works + +### Integration with PR Pipeline + +The regression tests are automatically run as part of every PR build, depending on the `EndToEndBuildTests` job for the F# compiler artifacts. + +### Template-Based Architecture + +The regression testing logic is implemented as a reusable Azure DevOps template that can be consumed by multiple pipelines: + +- **Template Location**: `eng/templates/regression-test-jobs.yml` +- **Integration**: Called from `azure-pipelines-PR.yml` +- **Dependencies**: Depends on `EndToEndBuildTests` job for compiler artifacts + +### Workflow + +1. **Build F# Compiler**: The `EndToEndBuildTests` job builds the F# compiler and publishes required artifacts +2. **Matrix Execution**: For each library in the test matrix (running in parallel): + - Checkout the third-party repository at a specific commit + - Install appropriate .NET SDK version using the repository's `global.json` + - Setup `Directory.Build.props` to import `UseLocalCompiler.Directory.Build.props` + - Build the library using its standard build script + - Publish MSBuild binary logs for analysis +3. **Report Results**: Success/failure status is reported with build logs for diagnosis + +### Key Features + +- **Reproducible Testing**: Uses specific commit SHAs for third-party libraries to ensure consistent results +- **Matrix Configuration**: Supports testing multiple libraries with different build requirements +- **Detailed Logging**: Captures comprehensive build logs, binary logs, and environment information +- **Artifact Publishing**: Publishes build outputs for analysis when builds fail + +## Current Test Matrix + +The pipeline currently tests against: + +| Library | Repository | Commit | Build Script | Purpose | +|---------|------------|--------|--------------|---------| +| FSharpPlus | fsprojects/FSharpPlus | f614035b75922aba41ed6a36c2fc986a2171d2b8 | build.cmd | Tests advanced F# language features | + +## Adding New Libraries + +To add a new library to the test matrix, update the template invocation in `azure-pipelines-PR.yml`: + +```yaml +# F# Compiler Regression Tests using third-party libraries +- template: /eng/templates/regression-test-jobs.yml + parameters: + testMatrix: + - repo: fsprojects/FSharpPlus + commit: f614035b75922aba41ed6a36c2fc986a2171d2b8 + buildScript: build.cmd + displayName: FSharpPlus + - repo: your-org/your-library # Add your library here + commit: abc123def456... # Specific commit SHA + buildScript: build.sh # Build script (build.cmd, build.sh, etc.) + displayName: YourLibrary # Human-readable name +``` + +Each test matrix entry requires: +- **repo**: GitHub repository in `owner/name` format +- **commit**: Specific commit SHA for reproducible results +- **buildScript**: Build script to execute (e.g., `build.cmd`, `build.sh`) +- **displayName**: Human-readable name for the job + +## Pipeline Configuration + +### Triggers + +Regression tests run automatically as part of PR builds when: +- **PR Pipeline**: Triggered by pull requests to main branches +- **Dependencies**: Runs after `EndToEndBuildTests` completes successfully +- **Parallel Execution**: Each repository in the test matrix runs as a separate job in parallel + +### Build Environment + +- **OS**: Windows (using `$(WindowsMachineQueueName)`) +- **Pool**: Standard public build pool (`$(DncEngPublicBuildPool)`) +- **Timeout**: 60 minutes per regression test job +- **.NET SDK**: Automatically detects and installs SDK version from each repository's `global.json` + +### Artifacts + +The regression tests publish focused artifacts for analysis: +- **FSharpCompilerArtifacts**: F# compiler build output (from `EndToEndBuildTests`) +- **UseLocalCompilerProps**: Configuration file for using local compiler (from `EndToEndBuildTests`) +- **{LibraryName}_BinaryLogs**: MSBuild binary logs from each tested library for efficient diagnosis + +## Troubleshooting Build Failures + +When a regression test fails: + +1. **Check the Job Summary**: Look at the final status report for high-level information. + +2. **Download Build Logs**: Download the published artifacts to examine detailed build output. + +3. **Compare Compiler Changes**: Review what changes were made to the compiler that might affect the failing library. + +4. **Local Reproduction**: Use the `UseLocalCompiler.Directory.Build.props` file to reproduce the issue locally. + +### Local Testing + +To test a library locally with your F# compiler build: + +1. Build the F# compiler: `.\Build.cmd -c Release -pack` + +2. In the third-party library directory, create a `Directory.Build.props`: + ```xml + + + + ``` + +3. Update the `LocalFSharpCompilerPath` in `UseLocalCompiler.Directory.Build.props` to point to your F# repository. + +4. Set environment variables: + ```cmd + set LoadLocalFSharpBuild=true + set LocalFSharpCompilerConfiguration=Release + ``` + +5. Run the library's build script. + +## Best Practices + +### For Library Selection + +- **Coverage**: Choose libraries that exercise different F# language features +- **Popularity**: Include widely-used libraries that represent real-world usage +- **Stability**: Use libraries with stable build processes and minimal external dependencies +- **Diversity**: Include libraries with different build systems and target frameworks + +### For Maintenance + +- **Regular Updates**: Periodically update commit SHAs to newer stable versions +- **Monitor Dependencies**: Watch for changes in third-party library build requirements +- **Baseline Management**: Update baselines when intentional breaking changes are made + +## Technical Details + +### UseLocalCompiler.Directory.Build.props + +This MSBuild props file configures projects to use the locally built F# compiler instead of the SDK version. Key settings: + +- `LocalFSharpCompilerPath`: Points to the F# compiler artifacts +- `DotnetFscCompilerPath`: Path to the fsc.dll compiler +- `DisableImplicitFSharpCoreReference`: Ensures local FSharp.Core is used + +### Path Handling + +The pipeline dynamically updates paths in the props file using PowerShell: +```powershell +$content -replace 'LocalFSharpCompilerPath.*MSBuildThisFileDirectory.*', 'LocalFSharpCompilerPath>$(Pipeline.Workspace)/FSharpCompiler<' +``` + +This ensures the correct path is used in the Azure DevOps environment. + +## Future Enhancements + +Potential improvements to the pipeline: + +1. **Performance Testing**: Measure compilation times and memory usage +2. **Multiple Target Frameworks**: Test libraries across different .NET versions +3. **Parallel Execution**: Run library tests in parallel for faster feedback +4. **Automatic Bisection**: Automatically identify which commit introduced a regression +5. **Integration with GitHub**: Post regression test results as PR comments \ No newline at end of file diff --git a/eng/templates/regression-test-jobs.yml b/eng/templates/regression-test-jobs.yml new file mode 100644 index 0000000000..d99841d01f --- /dev/null +++ b/eng/templates/regression-test-jobs.yml @@ -0,0 +1,236 @@ +# Template for F# Compiler Regression Tests +# Tests third-party F# projects with the freshly built compiler + +parameters: +- name: testMatrix + type: object + +jobs: +# Test against third-party repositories +- ${{ each item in parameters.testMatrix }}: + - job: RegressionTest_${{ replace(item.repo, '/', '_') }} + displayName: 'Regression Test: ${{ item.repo }}' + dependsOn: EndToEndBuildTests + pool: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals $(WindowsMachineQueueName) + timeoutInMinutes: 60 + variables: + TestRepoName: ${{ item.repo }} + TestCommit: ${{ item.commit }} + BuildScript: ${{ item.buildScript }} + DisplayName: ${{ item.displayName }} + steps: + - checkout: none + displayName: Skip default checkout + + # Download the F# compiler artifacts from EndToEndBuildTests job + - task: DownloadPipelineArtifact@2 + displayName: Download F# Compiler FSC Artifacts + inputs: + artifactName: 'FSharpCompilerFscArtifacts' + downloadPath: '$(Pipeline.Workspace)/FSharpCompiler/bin/fsc' + + - task: DownloadPipelineArtifact@2 + displayName: Download F# Core Artifacts + inputs: + artifactName: 'FSharpCoreArtifacts' + downloadPath: '$(Pipeline.Workspace)/FSharpCompiler/bin/FSharp.Core' + + - task: DownloadPipelineArtifact@2 + displayName: Download UseLocalCompiler props + inputs: + artifactName: 'UseLocalCompilerProps' + downloadPath: '$(Pipeline.Workspace)/Props' + + # Checkout the third-party repository at specific commit + - task: PowerShell@2 + displayName: 'Checkout $(DisplayName) at specific commit' + inputs: + script: | + Write-Host "Cloning repository: $(TestRepoName)" + git clone https://github.com/$(TestRepoName).git $(Pipeline.Workspace)/TestRepo + Set-Location $(Pipeline.Workspace)/TestRepo + + Write-Host "Checking out commit: $(TestCommit)" + git checkout $(TestCommit) + + Write-Host "Successfully checked out $(TestRepoName) at commit $(TestCommit)" + git log -1 --oneline + + Write-Host "Repository structure:" + Get-ChildItem -Name + + Write-Host "Verifying build script exists: $(BuildScript)" + if (Test-Path "$(BuildScript)") { + Write-Host "✓ Build script found: $(BuildScript)" + } else { + Write-Host "✗ Build script not found: $(BuildScript)" + Write-Host "Available files in root:" + Get-ChildItem + exit 1 + } + + # Install appropriate .NET SDK version using global.json if present + - task: UseDotNet@2 + displayName: 'Install .NET SDK for $(DisplayName)' + inputs: + packageType: sdk + useGlobalJson: true + workingDirectory: $(Pipeline.Workspace)/TestRepo + installationPath: $(Pipeline.Workspace)/TestRepo/.dotnet + + # Setup Directory.Build.props to import UseLocalCompiler configuration + - task: PowerShell@2 + displayName: 'Setup local compiler configuration for $(DisplayName)' + inputs: + script: | + Set-Location $(Pipeline.Workspace)/TestRepo + + # Create F# script to handle Directory.Build.props setup + $fsharpScript = @' + #r "nuget: System.Xml.ReaderWriter" + open System.IO + open System.Xml + + let useLocalCompilerImport = """""" + + let directoryBuildPropsPath = "Directory.Build.props" + + if File.Exists(directoryBuildPropsPath) then + printfn "Directory.Build.props exists, modifying it" + let doc = XmlDocument() + doc.Load(directoryBuildPropsPath) + + // Find the Project element + let projectElement = doc.SelectSingleNode("/Project") + if projectElement <> null then + // Check if our import already exists + let existingImport = doc.SelectSingleNode(sprintf "//Import[@Project='$(Pipeline.Workspace)/Props/UseLocalCompiler.Directory.Build.props']") + if existingImport = null then + let importElement = doc.CreateElement("Import") + importElement.SetAttribute("Project", "$(Pipeline.Workspace)/Props/UseLocalCompiler.Directory.Build.props") + projectElement.InsertBefore(importElement, projectElement.FirstChild) |> ignore + doc.Save(directoryBuildPropsPath) + printfn "Added UseLocalCompiler import to existing Directory.Build.props" + else + printfn "UseLocalCompiler import already exists" + else + printfn "Warning: Could not find Project element in Directory.Build.props" + else + printfn "Creating new Directory.Build.props" + let content = sprintf "\n %s\n" useLocalCompilerImport + File.WriteAllText(directoryBuildPropsPath, content) + + printfn "Directory.Build.props content:" + File.ReadAllText(directoryBuildPropsPath) |> printfn "%s" + '@ + + $fsharpScript | Out-File -FilePath "PrepareRepoForTesting.fsx" -Encoding UTF8 + + # Run the F# script using dotnet fsi + dotnet fsi PrepareRepoForTesting.fsx + + Write-Host "UseLocalCompiler.Directory.Build.props will be referenced from: $(Pipeline.Workspace)/Props/UseLocalCompiler.Directory.Build.props" + + # Report dotnet info in test environment + - task: PowerShell@2 + displayName: 'Report build environment for $(DisplayName)' + inputs: + script: | + Set-Location $(Pipeline.Workspace)/TestRepo + Write-Host "===========================================" + Write-Host "Environment Information for $(DisplayName)" + Write-Host "===========================================" + dotnet --info + Write-Host "" + Write-Host "MSBuild version:" + dotnet msbuild -version + Write-Host "" + Write-Host "F# Compiler artifacts available:" + Get-ChildItem "$(Pipeline.Workspace)\FSharpCompiler\bin\fsc\Release\net9.0" -Name + Write-Host "" + Write-Host "F# Core available:" + if (Test-Path "$(Pipeline.Workspace)\FSharpCompiler\bin\FSharp.Core\Release\netstandard2.0\FSharp.Core.dll") { + Write-Host "✓ FSharp.Core.dll found" + } else { + Write-Host "✗ FSharp.Core.dll not found" + } + Write-Host "" + Write-Host "Directory.Build.props content:" + Get-Content "Directory.Build.props" + Write-Host "" + Write-Host "===========================================" + + # Build the third-party project using local F# compiler + - task: PowerShell@2 + displayName: 'Build $(DisplayName) with local F# compiler' + env: + # Set environment variables to use local compiler and enforce binary logs + LocalFSharpCompilerPath: $(Pipeline.Workspace)/FSharpCompiler + LoadLocalFSharpBuild: true + LocalFSharpCompilerConfiguration: Release + # Force MSBuild binary logs + MSBUILDBINARYLOGGERENABLED: true + MSBUILDBINARYLOGGER: "*.binlog" + timeoutInMinutes: 45 + inputs: + script: | + Set-Location $(Pipeline.Workspace)/TestRepo + Write-Host "============================================" + Write-Host "Starting build for $(DisplayName)" + Write-Host "Repository: $(TestRepoName)" + Write-Host "Commit: $(TestCommit)" + Write-Host "Build Script: $(BuildScript)" + Write-Host "============================================" + Write-Host "" + + Write-Host "Executing: $(BuildScript)" + cmd /c "$(BuildScript)" + $exitCode = $LASTEXITCODE + + Write-Host "" + Write-Host "============================================" + Write-Host "Build completed for $(DisplayName)" + Write-Host "Exit code: $exitCode" + Write-Host "============================================" + + if ($exitCode -ne 0) { + exit $exitCode + } + + # Publish only MSBuild binary logs for efficient storage + - task: PublishPipelineArtifact@1 + displayName: 'Publish $(DisplayName) Binary Logs' + inputs: + targetPath: '$(Pipeline.Workspace)/TestRepo' + artifactName: '$(DisplayName)_BinaryLogs' + publishLocation: pipeline + condition: always() + continueOnError: true + + # Report success/failure + - task: PowerShell@2 + displayName: 'Report $(DisplayName) test result' + condition: always() + inputs: + script: | + Set-Location $(Pipeline.Workspace)/TestRepo + Write-Host "" + Write-Host "============================================" + Write-Host "Regression test completed for $(DisplayName)" + Write-Host "Repository: $(TestRepoName)" + Write-Host "Commit: $(TestCommit)" + Write-Host "Build Script: $(BuildScript)" + if ($env:AGENT_JOBSTATUS -eq "Succeeded") { + Write-Host "Status: ✓ SUCCESS" + Write-Host "The $(DisplayName) library builds successfully with the new F# compiler" + } else { + Write-Host "Status: ✗ FAILED" + Write-Host "The $(DisplayName) library failed to build with the new F# compiler" + Write-Host "Check the build logs and artifacts for details" + } + Write-Host "============================================" + + Write-Host "Binary logs found:" + Get-ChildItem "*.binlog" -ErrorAction SilentlyContinue | ForEach-Object { Write-Host $_.Name } \ No newline at end of file