diff --git a/azure-pipelines-PR.yml b/azure-pipelines-PR.yml
index 9f06d74493..014e5fb1d4 100644
--- a/azure-pipelines-PR.yml
+++ b/azure-pipelines-PR.yml
@@ -720,24 +720,48 @@ stages:
pool:
name: $(DncEngPublicBuildPool)
demands: ImageOverride -equals $(WindowsMachineQueueName)
- strategy:
- maxParallel: 2
- matrix:
- regular:
- _experimental_flag: ''
- experimental_features:
- _experimental_flag: ''
steps:
- checkout: self
clean: true
- script: .\Build.cmd -c Release -pack
env:
NativeToolsOnMachine: true
- FSHARP_EXPERIMENTAL_FEATURES: $(_experimental_flag)
- script: .\tests\EndToEndBuildTests\EndToEndBuildTests.cmd -c Release
- env:
- FSHARP_EXPERIMENTAL_FEATURES: $(_experimental_flag)
displayName: End to end build tests
+
+ # Publish artifacts for regression testing
+ - task: PublishPipelineArtifact@1
+ displayName: Publish F# Compiler FSC Artifacts for Regression Tests
+ inputs:
+ targetPath: '$(Build.SourcesDirectory)/artifacts/bin/fsc'
+ artifactName: 'FSharpCompilerFscArtifacts'
+ publishLocation: pipeline
+ condition: succeeded()
+
+ - task: PublishPipelineArtifact@1
+ displayName: Publish F# Core Artifacts for Regression Tests
+ inputs:
+ targetPath: '$(Build.SourcesDirectory)/artifacts/bin/FSharp.Core'
+ artifactName: 'FSharpCoreArtifacts'
+ publishLocation: pipeline
+ condition: succeeded()
+
+ - task: PublishPipelineArtifact@1
+ displayName: Publish UseLocalCompiler props file for Regression Tests
+ inputs:
+ targetPath: '$(Build.SourcesDirectory)/UseLocalCompiler.Directory.Build.props'
+ artifactName: 'UseLocalCompilerProps'
+ publishLocation: pipeline
+ condition: succeeded()
+
+ # F# Compiler Regression Tests using third-party libraries
+ - template: /eng/templates/regression-test-jobs.yml
+ parameters:
+ testMatrix:
+ - repo: fsprojects/FSharpPlus
+ commit: f614035b75922aba41ed6a36c2fc986a2171d2b8
+ buildScript: build.cmd
+ displayName: FSharpPlus
# Up-to-date - disabled due to it being flaky
#- job: UpToDate_Windows
diff --git a/docs/regression-testing-pipeline.md b/docs/regression-testing-pipeline.md
new file mode 100644
index 0000000000..7d617fb446
--- /dev/null
+++ b/docs/regression-testing-pipeline.md
@@ -0,0 +1,174 @@
+# F# Compiler Regression Testing
+
+This document describes the F# compiler regression testing functionality implemented as a reusable Azure DevOps template in `eng/templates/regression-test-jobs.yml` and integrated into the main PR pipeline (`azure-pipelines-PR.yml`).
+
+## Purpose
+
+The regression testing helps catch F# compiler regressions by building popular third-party F# libraries with the freshly built compiler from this repository. This provides early detection of breaking changes that might affect real-world F# projects.
+
+## How It Works
+
+### Integration with PR Pipeline
+
+The regression tests are automatically run as part of every PR build, depending on the `EndToEndBuildTests` job for the F# compiler artifacts.
+
+### Template-Based Architecture
+
+The regression testing logic is implemented as a reusable Azure DevOps template that can be consumed by multiple pipelines:
+
+- **Template Location**: `eng/templates/regression-test-jobs.yml`
+- **Integration**: Called from `azure-pipelines-PR.yml`
+- **Dependencies**: Depends on `EndToEndBuildTests` job for compiler artifacts
+
+### Workflow
+
+1. **Build F# Compiler**: The `EndToEndBuildTests` job builds the F# compiler and publishes required artifacts
+2. **Matrix Execution**: For each library in the test matrix (running in parallel):
+ - Checkout the third-party repository at a specific commit
+ - Install appropriate .NET SDK version using the repository's `global.json`
+ - Setup `Directory.Build.props` to import `UseLocalCompiler.Directory.Build.props`
+ - Build the library using its standard build script
+ - Publish MSBuild binary logs for analysis
+3. **Report Results**: Success/failure status is reported with build logs for diagnosis
+
+### Key Features
+
+- **Reproducible Testing**: Uses specific commit SHAs for third-party libraries to ensure consistent results
+- **Matrix Configuration**: Supports testing multiple libraries with different build requirements
+- **Detailed Logging**: Captures comprehensive build logs, binary logs, and environment information
+- **Artifact Publishing**: Publishes build outputs for analysis when builds fail
+
+## Current Test Matrix
+
+The pipeline currently tests against:
+
+| Library | Repository | Commit | Build Script | Purpose |
+|---------|------------|--------|--------------|---------|
+| FSharpPlus | fsprojects/FSharpPlus | f614035b75922aba41ed6a36c2fc986a2171d2b8 | build.cmd | Tests advanced F# language features |
+
+## Adding New Libraries
+
+To add a new library to the test matrix, update the template invocation in `azure-pipelines-PR.yml`:
+
+```yaml
+# F# Compiler Regression Tests using third-party libraries
+- template: /eng/templates/regression-test-jobs.yml
+ parameters:
+ testMatrix:
+ - repo: fsprojects/FSharpPlus
+ commit: f614035b75922aba41ed6a36c2fc986a2171d2b8
+ buildScript: build.cmd
+ displayName: FSharpPlus
+ - repo: your-org/your-library # Add your library here
+ commit: abc123def456... # Specific commit SHA
+ buildScript: build.sh # Build script (build.cmd, build.sh, etc.)
+ displayName: YourLibrary # Human-readable name
+```
+
+Each test matrix entry requires:
+- **repo**: GitHub repository in `owner/name` format
+- **commit**: Specific commit SHA for reproducible results
+- **buildScript**: Build script to execute (e.g., `build.cmd`, `build.sh`)
+- **displayName**: Human-readable name for the job
+
+## Pipeline Configuration
+
+### Triggers
+
+Regression tests run automatically as part of PR builds when:
+- **PR Pipeline**: Triggered by pull requests to main branches
+- **Dependencies**: Runs after `EndToEndBuildTests` completes successfully
+- **Parallel Execution**: Each repository in the test matrix runs as a separate job in parallel
+
+### Build Environment
+
+- **OS**: Windows (using `$(WindowsMachineQueueName)`)
+- **Pool**: Standard public build pool (`$(DncEngPublicBuildPool)`)
+- **Timeout**: 60 minutes per regression test job
+- **.NET SDK**: Automatically detects and installs SDK version from each repository's `global.json`
+
+### Artifacts
+
+The regression tests publish focused artifacts for analysis:
+- **FSharpCompilerArtifacts**: F# compiler build output (from `EndToEndBuildTests`)
+- **UseLocalCompilerProps**: Configuration file for using local compiler (from `EndToEndBuildTests`)
+- **{LibraryName}_BinaryLogs**: MSBuild binary logs from each tested library for efficient diagnosis
+
+## Troubleshooting Build Failures
+
+When a regression test fails:
+
+1. **Check the Job Summary**: Look at the final status report for high-level information.
+
+2. **Download Build Logs**: Download the published artifacts to examine detailed build output.
+
+3. **Compare Compiler Changes**: Review what changes were made to the compiler that might affect the failing library.
+
+4. **Local Reproduction**: Use the `UseLocalCompiler.Directory.Build.props` file to reproduce the issue locally.
+
+### Local Testing
+
+To test a library locally with your F# compiler build:
+
+1. Build the F# compiler: `.\Build.cmd -c Release -pack`
+
+2. In the third-party library directory, create a `Directory.Build.props`:
+ ```xml
+
+
+
+ ```
+
+3. Update the `LocalFSharpCompilerPath` in `UseLocalCompiler.Directory.Build.props` to point to your F# repository.
+
+4. Set environment variables:
+ ```cmd
+ set LoadLocalFSharpBuild=true
+ set LocalFSharpCompilerConfiguration=Release
+ ```
+
+5. Run the library's build script.
+
+## Best Practices
+
+### For Library Selection
+
+- **Coverage**: Choose libraries that exercise different F# language features
+- **Popularity**: Include widely-used libraries that represent real-world usage
+- **Stability**: Use libraries with stable build processes and minimal external dependencies
+- **Diversity**: Include libraries with different build systems and target frameworks
+
+### For Maintenance
+
+- **Regular Updates**: Periodically update commit SHAs to newer stable versions
+- **Monitor Dependencies**: Watch for changes in third-party library build requirements
+- **Baseline Management**: Update baselines when intentional breaking changes are made
+
+## Technical Details
+
+### UseLocalCompiler.Directory.Build.props
+
+This MSBuild props file configures projects to use the locally built F# compiler instead of the SDK version. Key settings:
+
+- `LocalFSharpCompilerPath`: Points to the F# compiler artifacts
+- `DotnetFscCompilerPath`: Path to the fsc.dll compiler
+- `DisableImplicitFSharpCoreReference`: Ensures local FSharp.Core is used
+
+### Path Handling
+
+The pipeline dynamically updates paths in the props file using PowerShell:
+```powershell
+$content -replace 'LocalFSharpCompilerPath.*MSBuildThisFileDirectory.*', 'LocalFSharpCompilerPath>$(Pipeline.Workspace)/FSharpCompiler<'
+```
+
+This ensures the correct path is used in the Azure DevOps environment.
+
+## Future Enhancements
+
+Potential improvements to the pipeline:
+
+1. **Performance Testing**: Measure compilation times and memory usage
+2. **Multiple Target Frameworks**: Test libraries across different .NET versions
+3. **Parallel Execution**: Run library tests in parallel for faster feedback
+4. **Automatic Bisection**: Automatically identify which commit introduced a regression
+5. **Integration with GitHub**: Post regression test results as PR comments
\ No newline at end of file
diff --git a/eng/templates/regression-test-jobs.yml b/eng/templates/regression-test-jobs.yml
new file mode 100644
index 0000000000..d99841d01f
--- /dev/null
+++ b/eng/templates/regression-test-jobs.yml
@@ -0,0 +1,236 @@
+# Template for F# Compiler Regression Tests
+# Tests third-party F# projects with the freshly built compiler
+
+parameters:
+- name: testMatrix
+ type: object
+
+jobs:
+# Test against third-party repositories
+- ${{ each item in parameters.testMatrix }}:
+ - job: RegressionTest_${{ replace(item.repo, '/', '_') }}
+ displayName: 'Regression Test: ${{ item.repo }}'
+ dependsOn: EndToEndBuildTests
+ pool:
+ name: $(DncEngPublicBuildPool)
+ demands: ImageOverride -equals $(WindowsMachineQueueName)
+ timeoutInMinutes: 60
+ variables:
+ TestRepoName: ${{ item.repo }}
+ TestCommit: ${{ item.commit }}
+ BuildScript: ${{ item.buildScript }}
+ DisplayName: ${{ item.displayName }}
+ steps:
+ - checkout: none
+ displayName: Skip default checkout
+
+ # Download the F# compiler artifacts from EndToEndBuildTests job
+ - task: DownloadPipelineArtifact@2
+ displayName: Download F# Compiler FSC Artifacts
+ inputs:
+ artifactName: 'FSharpCompilerFscArtifacts'
+ downloadPath: '$(Pipeline.Workspace)/FSharpCompiler/bin/fsc'
+
+ - task: DownloadPipelineArtifact@2
+ displayName: Download F# Core Artifacts
+ inputs:
+ artifactName: 'FSharpCoreArtifacts'
+ downloadPath: '$(Pipeline.Workspace)/FSharpCompiler/bin/FSharp.Core'
+
+ - task: DownloadPipelineArtifact@2
+ displayName: Download UseLocalCompiler props
+ inputs:
+ artifactName: 'UseLocalCompilerProps'
+ downloadPath: '$(Pipeline.Workspace)/Props'
+
+ # Checkout the third-party repository at specific commit
+ - task: PowerShell@2
+ displayName: 'Checkout $(DisplayName) at specific commit'
+ inputs:
+ script: |
+ Write-Host "Cloning repository: $(TestRepoName)"
+ git clone https://github.com/$(TestRepoName).git $(Pipeline.Workspace)/TestRepo
+ Set-Location $(Pipeline.Workspace)/TestRepo
+
+ Write-Host "Checking out commit: $(TestCommit)"
+ git checkout $(TestCommit)
+
+ Write-Host "Successfully checked out $(TestRepoName) at commit $(TestCommit)"
+ git log -1 --oneline
+
+ Write-Host "Repository structure:"
+ Get-ChildItem -Name
+
+ Write-Host "Verifying build script exists: $(BuildScript)"
+ if (Test-Path "$(BuildScript)") {
+ Write-Host "✓ Build script found: $(BuildScript)"
+ } else {
+ Write-Host "✗ Build script not found: $(BuildScript)"
+ Write-Host "Available files in root:"
+ Get-ChildItem
+ exit 1
+ }
+
+ # Install appropriate .NET SDK version using global.json if present
+ - task: UseDotNet@2
+ displayName: 'Install .NET SDK for $(DisplayName)'
+ inputs:
+ packageType: sdk
+ useGlobalJson: true
+ workingDirectory: $(Pipeline.Workspace)/TestRepo
+ installationPath: $(Pipeline.Workspace)/TestRepo/.dotnet
+
+ # Setup Directory.Build.props to import UseLocalCompiler configuration
+ - task: PowerShell@2
+ displayName: 'Setup local compiler configuration for $(DisplayName)'
+ inputs:
+ script: |
+ Set-Location $(Pipeline.Workspace)/TestRepo
+
+ # Create F# script to handle Directory.Build.props setup
+ $fsharpScript = @'
+ #r "nuget: System.Xml.ReaderWriter"
+ open System.IO
+ open System.Xml
+
+ let useLocalCompilerImport = """"""
+
+ let directoryBuildPropsPath = "Directory.Build.props"
+
+ if File.Exists(directoryBuildPropsPath) then
+ printfn "Directory.Build.props exists, modifying it"
+ let doc = XmlDocument()
+ doc.Load(directoryBuildPropsPath)
+
+ // Find the Project element
+ let projectElement = doc.SelectSingleNode("/Project")
+ if projectElement <> null then
+ // Check if our import already exists
+ let existingImport = doc.SelectSingleNode(sprintf "//Import[@Project='$(Pipeline.Workspace)/Props/UseLocalCompiler.Directory.Build.props']")
+ if existingImport = null then
+ let importElement = doc.CreateElement("Import")
+ importElement.SetAttribute("Project", "$(Pipeline.Workspace)/Props/UseLocalCompiler.Directory.Build.props")
+ projectElement.InsertBefore(importElement, projectElement.FirstChild) |> ignore
+ doc.Save(directoryBuildPropsPath)
+ printfn "Added UseLocalCompiler import to existing Directory.Build.props"
+ else
+ printfn "UseLocalCompiler import already exists"
+ else
+ printfn "Warning: Could not find Project element in Directory.Build.props"
+ else
+ printfn "Creating new Directory.Build.props"
+ let content = sprintf "\n %s\n" useLocalCompilerImport
+ File.WriteAllText(directoryBuildPropsPath, content)
+
+ printfn "Directory.Build.props content:"
+ File.ReadAllText(directoryBuildPropsPath) |> printfn "%s"
+ '@
+
+ $fsharpScript | Out-File -FilePath "PrepareRepoForTesting.fsx" -Encoding UTF8
+
+ # Run the F# script using dotnet fsi
+ dotnet fsi PrepareRepoForTesting.fsx
+
+ Write-Host "UseLocalCompiler.Directory.Build.props will be referenced from: $(Pipeline.Workspace)/Props/UseLocalCompiler.Directory.Build.props"
+
+ # Report dotnet info in test environment
+ - task: PowerShell@2
+ displayName: 'Report build environment for $(DisplayName)'
+ inputs:
+ script: |
+ Set-Location $(Pipeline.Workspace)/TestRepo
+ Write-Host "==========================================="
+ Write-Host "Environment Information for $(DisplayName)"
+ Write-Host "==========================================="
+ dotnet --info
+ Write-Host ""
+ Write-Host "MSBuild version:"
+ dotnet msbuild -version
+ Write-Host ""
+ Write-Host "F# Compiler artifacts available:"
+ Get-ChildItem "$(Pipeline.Workspace)\FSharpCompiler\bin\fsc\Release\net9.0" -Name
+ Write-Host ""
+ Write-Host "F# Core available:"
+ if (Test-Path "$(Pipeline.Workspace)\FSharpCompiler\bin\FSharp.Core\Release\netstandard2.0\FSharp.Core.dll") {
+ Write-Host "✓ FSharp.Core.dll found"
+ } else {
+ Write-Host "✗ FSharp.Core.dll not found"
+ }
+ Write-Host ""
+ Write-Host "Directory.Build.props content:"
+ Get-Content "Directory.Build.props"
+ Write-Host ""
+ Write-Host "==========================================="
+
+ # Build the third-party project using local F# compiler
+ - task: PowerShell@2
+ displayName: 'Build $(DisplayName) with local F# compiler'
+ env:
+ # Set environment variables to use local compiler and enforce binary logs
+ LocalFSharpCompilerPath: $(Pipeline.Workspace)/FSharpCompiler
+ LoadLocalFSharpBuild: true
+ LocalFSharpCompilerConfiguration: Release
+ # Force MSBuild binary logs
+ MSBUILDBINARYLOGGERENABLED: true
+ MSBUILDBINARYLOGGER: "*.binlog"
+ timeoutInMinutes: 45
+ inputs:
+ script: |
+ Set-Location $(Pipeline.Workspace)/TestRepo
+ Write-Host "============================================"
+ Write-Host "Starting build for $(DisplayName)"
+ Write-Host "Repository: $(TestRepoName)"
+ Write-Host "Commit: $(TestCommit)"
+ Write-Host "Build Script: $(BuildScript)"
+ Write-Host "============================================"
+ Write-Host ""
+
+ Write-Host "Executing: $(BuildScript)"
+ cmd /c "$(BuildScript)"
+ $exitCode = $LASTEXITCODE
+
+ Write-Host ""
+ Write-Host "============================================"
+ Write-Host "Build completed for $(DisplayName)"
+ Write-Host "Exit code: $exitCode"
+ Write-Host "============================================"
+
+ if ($exitCode -ne 0) {
+ exit $exitCode
+ }
+
+ # Publish only MSBuild binary logs for efficient storage
+ - task: PublishPipelineArtifact@1
+ displayName: 'Publish $(DisplayName) Binary Logs'
+ inputs:
+ targetPath: '$(Pipeline.Workspace)/TestRepo'
+ artifactName: '$(DisplayName)_BinaryLogs'
+ publishLocation: pipeline
+ condition: always()
+ continueOnError: true
+
+ # Report success/failure
+ - task: PowerShell@2
+ displayName: 'Report $(DisplayName) test result'
+ condition: always()
+ inputs:
+ script: |
+ Set-Location $(Pipeline.Workspace)/TestRepo
+ Write-Host ""
+ Write-Host "============================================"
+ Write-Host "Regression test completed for $(DisplayName)"
+ Write-Host "Repository: $(TestRepoName)"
+ Write-Host "Commit: $(TestCommit)"
+ Write-Host "Build Script: $(BuildScript)"
+ if ($env:AGENT_JOBSTATUS -eq "Succeeded") {
+ Write-Host "Status: ✓ SUCCESS"
+ Write-Host "The $(DisplayName) library builds successfully with the new F# compiler"
+ } else {
+ Write-Host "Status: ✗ FAILED"
+ Write-Host "The $(DisplayName) library failed to build with the new F# compiler"
+ Write-Host "Check the build logs and artifacts for details"
+ }
+ Write-Host "============================================"
+
+ Write-Host "Binary logs found:"
+ Get-ChildItem "*.binlog" -ErrorAction SilentlyContinue | ForEach-Object { Write-Host $_.Name }
\ No newline at end of file