Add Azure DevOps template for F# compiler regression testing

azure-pipelines-regression-test.yml

@@ -0,0 +1,348 @@
+# Regression Testing Pipeline
+# Tests F# compiler regressions by building third-party F# projects with the freshly built compiler
+
+trigger:
+  branches:
+    include:
+    - main
+    - release/*
+    - feature/*
+  paths:
+    include:
+    - src/*
+    exclude:
+    - docs/*
+    - .github/*
+    - '*.md'
+
+pr:
+  branches:
+    include:
+    - main
+    - release/*
+    - feature/*
+  paths:
+    include:
+    - src/*
+    exclude:
+    - docs/*
+    - .github/*
+    - '*.md'
+
+variables:
+  - name: _TeamName
+    value: FSharp
+  - name: _BuildConfig
+    value: Release
+  - name: _PublishUsingPipelines
+    value: true
+  - name: Codeql.Enabled
+    value: false  # Disabled for regression tests
+  # Pick up pool provider name behavior from shared yaml template
+  - template: /eng/common/templates/variables/pool-providers.yml
+
+# Matrix of third-party repositories to test against
+# This configuration can be extended in the future
+parameters:
+- name: testMatrix
+  type: object
+  default:
+  - repo: fsprojects/FSharpPlus
+    commit: f614035b75922aba41ed6a36c2fc986a2171d2b8
+    buildScript: build.cmd
+    displayName: FSharpPlus
+
+stages:
+- stage: RegressionTest
+  displayName: F# Compiler Regression Tests
+  jobs:
+  - job: BuildCompiler
+    displayName: Build F# Compiler
+    pool:
+      name: $(DncEngPublicBuildPool)
+      demands: ImageOverride -equals $(WindowsMachineQueueName)
+    timeoutInMinutes: 60
+    steps:
+    - checkout: self
+      clean: true
+      displayName: Checkout F# compiler source
+
+    - task: UseDotNet@2
+      displayName: install SDK
+      inputs:
+        packageType: sdk
+        version: '10.x'
+        includePreviewVersions: true
+        workingDirectory: $(Build.SourcesDirectory)
+        installationPath: $(Build.SourcesDirectory)/.dotnet
+
+    - script: .\eng\common\dotnet.cmd
+      displayName: Ensure correct .NET SDK version
+
+    - script: .\Build.cmd -c $(_BuildConfig) -pack
+      env:
+        NativeToolsOnMachine: true
+      displayName: Build F# compiler and create packages
+
+    - task: PublishPipelineArtifact@1
+      displayName: Publish F# Compiler Artifacts
+      inputs:
+        targetPath: '$(Build.SourcesDirectory)/artifacts'
+        artifactName: 'FSharpCompilerArtifacts'
+        publishLocation: pipeline
+      condition: succeeded()
+
+    - task: PublishPipelineArtifact@1
+      displayName: Publish UseLocalCompiler props file
+      inputs:
+        targetPath: '$(Build.SourcesDirectory)/UseLocalCompiler.Directory.Build.props'
+        artifactName: 'UseLocalCompilerProps'
+        publishLocation: pipeline
+      condition: succeeded()
+
+  # Test against third-party repositories
+  - ${{ each item in parameters.testMatrix }}:
+    - job: Test_${{ replace(item.repo, '/', '_') }}
+      displayName: 'Regression Test: ${{ item.displayName }}'
+      dependsOn: BuildCompiler
+      pool:
+        name: $(DncEngPublicBuildPool)
+        demands: ImageOverride -equals $(WindowsMachineQueueName)
+      timeoutInMinutes: 60
+      variables:
+        TestRepoName: ${{ item.repo }}
+        TestCommit: ${{ item.commit }}
+        BuildScript: ${{ item.buildScript }}
+        DisplayName: ${{ item.displayName }}
+      steps:
+      - checkout: none
+        displayName: Skip default checkout
+
+      # Download the F# compiler artifacts
+      - task: DownloadPipelineArtifact@2
+        displayName: Download F# Compiler Artifacts
+        inputs:
+          artifactName: 'FSharpCompilerArtifacts'
+          downloadPath: '$(Pipeline.Workspace)/FSharpCompiler'
+
+      - task: DownloadPipelineArtifact@2
+        displayName: Download UseLocalCompiler props
+        inputs:
+          artifactName: 'UseLocalCompilerProps'
+          downloadPath: '$(Pipeline.Workspace)/Props'
+
+      # Checkout the third-party repository at specific commit
+      - task: PowerShell@2
+        displayName: 'Checkout $(DisplayName) at specific commit'
+        inputs:
+          script: |
+            Write-Host "Cloning repository: $(TestRepoName)"
+            git clone https://github.com/$(TestRepoName).git $(Pipeline.Workspace)/TestRepo
+            Set-Location $(Pipeline.Workspace)/TestRepo
+
+            Write-Host "Checking out commit: $(TestCommit)"
+            git checkout $(TestCommit)
+
+            Write-Host "Successfully checked out $(TestRepoName) at commit $(TestCommit)"
+            git log -1 --oneline
+
+            Write-Host "Repository structure:"
+            Get-ChildItem -Name
+
+            Write-Host "Verifying build script exists: $(BuildScript)"
+            if (Test-Path "$(BuildScript)") {
+              Write-Host "✓ Build script found: $(BuildScript)"
+            } else {
+              Write-Host "✗ Build script not found: $(BuildScript)"
+              Write-Host "Available files in root:"
+              Get-ChildItem
+              exit 1
+            }
+
+      # Detect .NET SDK version from global.json if it exists
+      - task: PowerShell@2
+        displayName: 'Detect .NET SDK version for $(DisplayName)'
+        inputs:
+          script: |
+            Set-Location $(Pipeline.Workspace)/TestRepo
+
+            if (Test-Path "global.json") {
+              Write-Host "Found global.json, reading .NET SDK version"
+              $globalJson = Get-Content "global.json" | ConvertFrom-Json
+              $sdkVersion = $globalJson.sdk.version
+              Write-Host "Detected .NET SDK version: $sdkVersion"
+              Write-Host "##vso[task.setvariable variable=DotNetSdkVersion]$sdkVersion"
+            } else {
+              Write-Host "No global.json found, will use latest public SDK"
+              Write-Host "##vso[task.setvariable variable=DotNetSdkVersion]10.x"
+            }
+
+      # Install appropriate .NET SDK version
+      - task: UseDotNet@2
+        displayName: 'Install .NET SDK for $(DisplayName)'
+        inputs:
+          packageType: sdk
+          version: $(DotNetSdkVersion)
+          includePreviewVersions: true
+          workingDirectory: $(Pipeline.Workspace)/TestRepo
+          installationPath: $(Pipeline.Workspace)/TestRepo/.dotnet
+
+      # Setup Directory.Build.props and UseLocalCompiler configuration using F# script
+      - task: PowerShell@2
+        displayName: 'Setup local compiler configuration for $(DisplayName)'
+        inputs:
+          script: |
+            Set-Location $(Pipeline.Workspace)/TestRepo
+
+            # Create F# script to handle Directory.Build.props setup
+            $fsharpScript = @'
+            #r "nuget: System.Xml.ReaderWriter"
+            open System.IO
+            open System.Xml
+
+            let useLocalCompilerImport = """<Import Project="UseLocalCompiler.Directory.Build.props" />"""
+
+            let directoryBuildPropsPath = "Directory.Build.props"
+
+            if File.Exists(directoryBuildPropsPath) then
+                printfn "Directory.Build.props exists, modifying it"
+                let doc = XmlDocument()
+                doc.Load(directoryBuildPropsPath)
+
+                // Find the Project element
+                let projectElement = doc.SelectSingleNode("/Project")
+                if projectElement <> null then
+                    // Check if our import already exists
+                    let existingImport = doc.SelectSingleNode(sprintf "//Import[@Project='UseLocalCompiler.Directory.Build.props']")
+                    if existingImport = null then
+                        let importElement = doc.CreateElement("Import")
+                        importElement.SetAttribute("Project", "UseLocalCompiler.Directory.Build.props")
+                        projectElement.InsertBefore(importElement, projectElement.FirstChild) |> ignore
+                        doc.Save(directoryBuildPropsPath)
+                        printfn "Added UseLocalCompiler import to existing Directory.Build.props"
+                    else
+                        printfn "UseLocalCompiler import already exists"
+                else
+                    printfn "Warning: Could not find Project element in Directory.Build.props"
+            else
+                printfn "Creating new Directory.Build.props"
+                let content = sprintf "<Project>\n  %s\n</Project>" useLocalCompilerImport
+                File.WriteAllText(directoryBuildPropsPath, content)
+
+            printfn "Directory.Build.props content:"
+            File.ReadAllText(directoryBuildPropsPath) |> printfn "%s"
+            '@
+
+            $fsharpScript | Out-File -FilePath "PrepareRepoForTesting.fsx" -Encoding UTF8
+
+            # Run the F# script using dotnet fsi
+            dotnet fsi PrepareRepoForTesting.fsx
+
+            # Copy UseLocalCompiler.Directory.Build.props as-is (path will be set via environment variable)
+            Copy-Item "$(Pipeline.Workspace)/Props/UseLocalCompiler.Directory.Build.props" "UseLocalCompiler.Directory.Build.props" -Force
+
+            Write-Host "UseLocalCompiler.Directory.Build.props content (first 20 lines):"
+            Get-Content "UseLocalCompiler.Directory.Build.props" | Select-Object -First 20
+
+      # Report dotnet info in test environment
+      - task: PowerShell@2
+        displayName: 'Report build environment for $(DisplayName)'
+        inputs:
+          script: |
+            Set-Location $(Pipeline.Workspace)/TestRepo
+            Write-Host "==========================================="
+            Write-Host "Environment Information for $(DisplayName)"
+            Write-Host "==========================================="
+            dotnet --info
+            Write-Host ""
+            Write-Host "MSBuild version:"
+            dotnet msbuild -version
+            Write-Host ""
+            Write-Host "F# Compiler artifacts available:"
+            Get-ChildItem "$(Pipeline.Workspace)\FSharpCompiler\bin\fsc\Release\net9.0" -Name
+            Write-Host ""
+            Write-Host "F# Core available:"
+            if (Test-Path "$(Pipeline.Workspace)\FSharpCompiler\bin\FSharp.Core\Release\netstandard2.0\FSharp.Core.dll") {
+              Write-Host "✓ FSharp.Core.dll found"
+            } else {
+              Write-Host "✗ FSharp.Core.dll not found"
+            }
+            Write-Host ""
+            Write-Host "Directory.Build.props content:"
+            Get-Content "Directory.Build.props"
+            Write-Host ""
+            Write-Host "==========================================="
+
+      # Build the third-party project using local F# compiler
+      - task: PowerShell@2
+        displayName: 'Build $(DisplayName) with local F# compiler'
+        env:
+          # Set environment variables to use local compiler and enforce binary logs
+          LocalFSharpCompilerPath: $(Pipeline.Workspace)/FSharpCompiler
+          LoadLocalFSharpBuild: true
+          LocalFSharpCompilerConfiguration: Release
+          # Force MSBuild binary logs
+          MSBUILDBINARYLOGGERENABLED: true
+          MSBUILDBINARYLOGGER: "*.binlog"
+        timeoutInMinutes: 45
+        inputs:
+          script: |
+            Set-Location $(Pipeline.Workspace)/TestRepo
+            Write-Host "============================================"
+            Write-Host "Starting build for $(DisplayName)"
+            Write-Host "Repository: $(TestRepoName)"
+            Write-Host "Commit: $(TestCommit)"
+            Write-Host "Build Script: $(BuildScript)"
+            Write-Host "============================================"
+            Write-Host ""
+
+            Write-Host "Executing: $(BuildScript)"
+            cmd /c "$(BuildScript)"
+            $exitCode = $LASTEXITCODE
+
+            Write-Host ""
+            Write-Host "============================================"
+            Write-Host "Build completed for $(DisplayName)"
+            Write-Host "Exit code: $exitCode"
+            Write-Host "============================================"
+
+            if ($exitCode -ne 0) {
+              exit $exitCode
+            }
+
+      # Capture and publish build logs on failure
+      # Publish only MSBuild binary logs
+      - task: PublishPipelineArtifact@1
+        displayName: 'Publish $(DisplayName) Binary Logs'
+        inputs:
+          targetPath: '$(Pipeline.Workspace)/TestRepo'
+          artifactName: '$(DisplayName)_BinaryLogs'
+          publishLocation: pipeline
+        condition: always()
+        continueOnError: true
+
+      # Report success/failure
+      - task: PowerShell@2
+        displayName: 'Report $(DisplayName) test result'
+        condition: always()
+        inputs:
+          script: |
+            Set-Location $(Pipeline.Workspace)/TestRepo
+            Write-Host ""
+            Write-Host "============================================"
+            Write-Host "Regression test completed for $(DisplayName)"
+            Write-Host "Repository: $(TestRepoName)"
+            Write-Host "Commit: $(TestCommit)"
+            Write-Host "Build Script: $(BuildScript)"
+            if ($env:AGENT_JOBSTATUS -eq "Succeeded") {
+              Write-Host "Status: ✓ SUCCESS"
+              Write-Host "The $(DisplayName) library builds successfully with the new F# compiler"
+            } else {
+              Write-Host "Status: ✗ FAILED"
+              Write-Host "The $(DisplayName) library failed to build with the new F# compiler"
+              Write-Host "Check the build logs and artifacts for details"
+            }
+            Write-Host "============================================"
+
+            Write-Host "Binary logs found:"
+            Get-ChildItem "*.binlog" -ErrorAction SilentlyContinue | ForEach-Object { Write-Host $_.Name }