Use a dedicated APIScan pipeline with 'NonOfficial' pipeline template

daxian-dbw · daxian-dbw · commit 6e5563f3a6a6 · 2025-09-05T10:59:42.000-07:00
diff --git a/.pipelines/APIScan.yml b/.pipelines/APIScan.yml
@@ -0,0 +1,101 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+name: apiscan-$(BUILD.SOURCEBRANCHNAME)-$(Build.BuildId)
+trigger: none
+
+parameters:
+  - name: FORCE_CODEQL
+    displayName: Debugging - Enable CodeQL and set cadence to 1 hour
+    type: boolean
+    default: false
+  - name: SkipVerifyPackages
+    type: boolean
+    default: false
+
+variables:
+  # PAT permissions NOTE: Declare a SymbolServerPAT variable in this group with a 'microsoft' organizanization scoped PAT with 'Symbols' Read permission.
+  # A PAT in the wrong org will give a single Error 203. No PAT will give a single Error 401, and individual pdbs may be missing even if permissions are correct.
+  - group: symbols
+  - name: ob_outputDirectory
+    value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT'
+  - name: CDP_DEFINITION_BUILD_COUNT
+    value: $[counter('', 0)]
+  # Defines the variables CgPat, CgOrganization, and CgProject
+  - group: 'ComponentGovernance'
+  - group: 'PoolNames'
+  - name: LinuxContainerImage
+    value: mcr.microsoft.com/onebranch/azurelinux/build:3.0
+  - name: WindowsContainerImage
+    value: onebranch.azurecr.io/windows/ltsc2022/vse2022:latest
+  - ${{ if eq(parameters['FORCE_CODEQL'],'true') }}:
+    # Cadence is hours before CodeQL will allow a re-upload of the database
+    - name: CodeQL.Cadence
+      value: 0
+  - name: CODEQL_ENABLED
+    ${{ if or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), eq(parameters['FORCE_CODEQL'],'true')) }}:
+      value: true
+    ${{ else }}:
+      value: false
+  - name: Codeql.TSAEnabled
+    value: $(CODEQL_ENABLED)
+  # AnalyzeInPipeline: false = upload results
+  # AnalyzeInPipeline: true = do not upload results
+  - name: Codeql.AnalyzeInPipeline
+    value: $(CODEQL_ENABLED)
+
+resources:
+  repositories:
+  - repository: templates
+    type: git
+    name: OneBranch.Pipelines/GovernedTemplates
+    ref: refs/heads/main
+
+extends:
+  template: v2/OneBranch.NonOfficial.CrossPlat.yml@templates
+  parameters:
+    featureFlags:
+      WindowsHostVersion:
+        Version: 2022
+    globalSdl:
+      codeql:
+        compiled:
+          enabled: $(CODEQL_ENABLED)
+        tsaEnabled: $(CODEQL_ENABLED) # This enables TSA bug filing only for CodeQL 3000
+      armory:
+        enabled: false
+      sbom:
+        enabled: false
+      cg:
+        enabled: true
+        ignoreDirectories: 'docs,shell,tools'
+      tsa:
+        enabled: true  # onebranch publish all SDL results to TSA. If TSA is disabled all SDL tools will forced into 'break' build mode.
+      credscan:
+        enabled: true
+        scanFolder:  $(Build.SourcesDirectory)
+      binskim:
+        break: true  # always break the build on binskim issues in addition to TSA upload
+      policheck:
+        break: true  # always break the build on policheck issues. You can disable it by setting to 'false'
+      # APIScan requires a non-Ready-To-Run build
+      apiscan:
+        enabled: true
+        softwareName: 'AIShell' # Default is repo name
+        versionNumber: '1.0' # Default is build number
+        isLargeApp: false # Default: false.
+        symbolsFolder: $(SymbolsServerUrl);$(ob_outputDirectory)
+      tsaOptionsFile: .config\tsaoptions.json
+      psscriptanalyzer:
+          enabled: true
+          policyName: Microsoft
+          break: false
+
+    stages:
+      - stage: APIScan
+        displayName: 'ApiScan'
+        dependsOn: []
+        jobs:
+          - template: /.pipelines/templates/compliance/apiscan-build.yaml@self
+            parameters:
+              parentJobs: []
diff --git a/.pipelines/Build-Official.yml b/.pipelines/Build-Official.yml
@@ -112,8 +112,3 @@ extends:
       displayName: module - build and sign
       jobs:
       - template: /.pipelines/templates/module-build.yml@self
-
-    - stage: APIScan
-      displayName: 'ApiScan'
-      jobs:
-      - template: /.pipelines/templates/apiscan.yaml@self
diff --git a/.pipelines/templates/compliance/apiscan-build.yaml b/.pipelines/templates/compliance/apiscan-build.yaml
@@ -12,32 +12,17 @@ jobs:
     value : false
   - name: NugetSecurityAnalysisWarningLevel
     value: none
-  # PAT permissions NOTE: Declare a SymbolServerPAT variable in this group with a 'microsoft' organizanization scoped PAT with 'Symbols' Read permission.
-  # A PAT in the wrong org will give a single Error 203. No PAT will give a single Error 401, and individual pdbs may be missing even if permissions are correct.
-  - group: symbols
   # Defines the variables APIScanClient, APIScanTenant and APIScanSecret
   - group: PS-PS-APIScan
-  - group: DotNetPrivateBuildAccess
   - group: 'Azure Blob variable group'
-  - group: ReleasePipelineSecrets
   - group: mscodehub-feed-read-general
   - group: mscodehub-feed-read-akv
   - name: ob_outputDirectory
-    value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT'
+    value: '$(Build.ArtifactStagingDirectory)\ONEBRANCH_ARTIFACT'
   - name: repoRoot
     value: $(Build.SourcesDirectory)\AIShell
   - name: ob_sdl_tsa_configFile
     value: $(repoRoot)\.config\tsaoptions.json
-  - name: ob_sdl_apiscan_enabled
-    value: true
-  - name: ob_sdl_apiscan_softwareName
-    value: 'AIShell'
-  - name: ob_sdl_apiscan_versionNumber
-    value: '1.0'
-  - name: ob_sdl_apiscan_isLargeApp
-    value: false
-  - name: ob_sdl_apiscan_symbolsFolder
-    value: $(SymbolsServerUrl);$(ob_outputDirectory)
   - name: Codeql.SourceRoot
     value: $(repoRoot)
 
@@ -54,7 +39,7 @@ jobs:
     env:
       ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase
 
-  - template: update-nuget-config.yml@self
+  - template: ..\update-nuget-config.yml@self
     parameters:
       repoRoot: $(repoRoot)
 
@@ -65,13 +50,13 @@ jobs:
       packageType: 'sdk'
       workingDirectory: $(Build.SourcesDirectory)"
 
-  - pwsh: |
-      dotnet tool install dotnet-symbol --tool-path $(Agent.ToolsDirectory)\tools\dotnet-symbol
-      $symbolToolPath = Get-ChildItem -Path $(Agent.ToolsDirectory)\tools\dotnet-symbol\dotnet-symbol.exe | Select-Object -First 1 -ExpandProperty FullName
-      Write-Host "##vso[task.setvariable variable=symbolToolPath]$symbolToolPath"
-    displayName: Install dotnet-symbol
-    workingDirectory: '$(repoRoot)'
-    retryCountOnTaskFailure: 2
+  # - pwsh: |
+  #     dotnet tool install dotnet-symbol --tool-path $(Agent.ToolsDirectory)\tools\dotnet-symbol
+  #     $symbolToolPath = Get-ChildItem -Path $(Agent.ToolsDirectory)\tools\dotnet-symbol\dotnet-symbol.exe | Select-Object -First 1 -ExpandProperty FullName
+  #     Write-Host "##vso[task.setvariable variable=symbolToolPath]$symbolToolPath"
+  #   displayName: Install dotnet-symbol
+  #   workingDirectory: '$(repoRoot)'
+  #   retryCountOnTaskFailure: 2
 
   - task: AzurePowerShell@5
     displayName: Download winverify-private Artifacts
@@ -85,15 +70,15 @@ jobs:
         # download smybols for getfilesiginforedist.dll
         $storageAccountName = "pscoretestdata"
         $containerName = 'winverify-private'
-        $winverifySymbolsPath = New-Item -ItemType Directory -Path '$(System.ArtifactsDirectory)/winverify-symbols' -Force
+        $winverifySymbolsPath = New-Item -ItemType Directory -Path '$(System.ArtifactsDirectory)\winverify-symbols' -Force
         $dllName = 'getfilesiginforedist.dll'
         $winverifySymbolsDllPath = Join-Path $winverifySymbolsPath $dllName
 
         $context = New-AzStorageContext -StorageAccountName $storageAccountName -UseConnectedAccount
         Get-AzStorageBlobContent -Container $containerName -Blob $dllName -Destination $winverifySymbolsDllPath -Context $context
 
   - pwsh: |
-      Get-ChildItem -Path '$(System.ArtifactsDirectory)/winverify-symbols'
+      Get-ChildItem -Path '$(System.ArtifactsDirectory)\winverify-symbols'
     displayName: Capture winverify-private Artifacts
     workingDirectory: '$(repoRoot)'
     condition: succeededOrFailed()
@@ -105,42 +90,70 @@ jobs:
       Language: csharp
 
   - pwsh: |
-      Import-Module $(repoRoot)/build.psm1 -Force
+      Import-Module '$(repoRoot)\build.psm1' -Force
       Start-Build -Configuration StaticAnalysis -Runtime fxdependent -Clean -Verbose
 
-      $outputJson = '$(repoRoot)/_build_output_.json'
+      $outputJson = '$(repoRoot)\_build_output_.json'
       if (-not (Test-Path $outputJson)) {
         throw "'_build_output_.json' was not produced."
       }
 
       $result = Get-Content $outputJson | ConvertFrom-Json
-      Write-Verbose "App path: $($result.App)" -Verbose
       $OutputFolder = $result.App
+      Write-Verbose "App path: $OutputFolder" -Verbose
+
+      Write-Verbose -Verbose -Message "Deleting the 'ref' folder ..."
+      if (Test-Path $OutputFolder\ref) {
+        Remove-Item -Recurse -Force $OutputFolder\ref
+      }
 
-      Write-Verbose -Verbose -Message "Deleting ref folder from output folder"
-      if (Test-Path $OutputFolder/ref) {
-        Remove-Item -Recurse -Force $OutputFolder/ref
+      # Only keep windows runtimes
+      Write-Verbose -Verbose -Message "Deleting non-win-x64 runtimes ..."
+      Get-ChildItem -Path "$OutputFolder\runtimes\*" | Where-Object {$_.FullName -notmatch '.*\\runtimes\\win'} | Foreach-Object {
+        Write-Verbose -Verbose -Message "Deleting $($_.FullName)"
+        Remove-Item -Path $_.FullName -Recurse -Force
       }
 
-      Copy-Item -Path "$OutputFolder\*" -Destination '$(ob_outputDirectory)' -Recurse -Verbose
+      # Remove win-x86/arm/arm64 runtimes due to issues with those runtimes
+      Write-Verbose -Verbose -Message "Temporarily deleting win-x86/arm/arm64 runtimes ..."
+      Get-ChildItem -Path "$OutputFolder\runtimes\*" | Where-Object {$_.FullName -match '.*\\runtimes\\win-(x86|arm)'} | Foreach-Object {
+        Write-Verbose -Verbose -Message "Deleting $($_.FullName)"
+        Remove-Item -Path $_.FullName -Recurse -Force
+      }
+
+      Write-Host
+      Write-Verbose -Verbose -Message "Show content in 'runtimes' folder:"
+      Get-ChildItem -Path "$OutputFolder\runtimes"
+      Write-Host
+
+      # Replace 'getfilesiginforedist.dll' from the 'runtimes\win-x64' folder
+      Write-Verbose -Verbose -Message "Replace 'getfilesiginforedist.dll':"
+      $targetFile = Get-ChildItem -Path "$OutputFolder\runtimes\*" -Recurse | Where-Object {$_.Name -eq 'getfilesiginforedist.dll'}
+      Remove-Item -Path $targetFile.FullName -Verbose
+      Copy-Item -Path '$(System.ArtifactsDirectory)\winverify-symbols\getfilesiginforedist.dll' -Destination $targetFile.FullName -Verbose
+
+      Write-Host "##vso[task.setvariable variable=appPath]$OutputFolder"
     workingDirectory: '$(repoRoot)'
     displayName: 'Build AIShell Source'
 
   - pwsh: |
-      # Only key windows runtimes
-      Get-ChildItem -Path '$(ob_outputDirectory)\runtimes\*' -File -Recurse | Where-Object {$_.FullName -notmatch '.*\/runtimes\/win'} | Foreach-Object {
-        Write-Verbose -Verbose -Message "Deleting $($_.FullName)"
-        Remove-Item -Force -Verbose -Path $_.FullName
+      $Destination = '$(ob_outputDirectory)'
+      if (-not (Test-Path $Destination)) {
+        Write-Verbose -Verbose -Message "Creating destination folder '$Destination'"
+        $null = mkdir $Destination
       }
 
-      # Temporarily remove runtimes/win-x86 due to issues with that runtime
-      Get-ChildItem -Path '$(ob_outputDirectory)\runtimes\*' -File -Recurse | Where-Object {$_.FullName -match '.*\/runtimes\/win-x86\/'} | Foreach-Object {
-        Write-Verbose -Verbose -Message "Deleting $($_.FullName)"
-        Remove-Item -Force -Verbose -Path $_.FullName
-      }
+      Write-Verbose -Verbose -Message "Copy AIShell app to '$Destination':"
+      Copy-Item -Path '$(appPath)\*' -Destination $Destination -Recurse -Verbose
 
+      Write-Host
+      Write-Verbose -Verbose -Message "Show content in '$Destination':"
+      Get-ChildItem -Path $Destination | Out-String -width 150
+      Write-Host
+      Write-Verbose -Verbose -Message "Show content in '$Destination\runtimes':"
+      Get-ChildItem -Path "$Destination\runtimes" -Recurse | Out-String -width 150
     workingDirectory: '$(repoRoot)'
-    displayName: 'Remove unused runtimes'
+    displayName: 'Copy AIShell to ob_outputDirectory'
 
   - task: CodeQL3000Finalize@0 # Add CodeQL Finalize task right after your 'Build' step.
     displayName: 🔏 CodeQL 3000 Finalize
@@ -153,16 +166,17 @@ jobs:
     condition: succeededOrFailed()
 
   # Explicitly download symbols for the drop since the SDL image doesn't have http://SymWeb access and APIScan cannot handle https yet.
-  - pwsh: |
-      $pat = '$(SymbolServerPAT)'
-      if ($pat -like '*PAT*' -or $pat -eq '') {
-        throw 'No PAT defined'
-      }
-      $url = 'https://microsoft.artifacts.visualstudio.com/defaultcollection/_apis/symbol/symsrv'
-      $(symbolToolPath) --authenticated-server-path $(SymbolServerPAT) $url --symbols -d "$env:ob_outputDirectory\*" --recurse-subdirectories
-    displayName: 'Download Symbols for binaries'
-    retryCountOnTaskFailure: 2
-    workingDirectory: '$(repoRoot)'
+  # - pwsh: |
+  #     $pat = '$(SymbolServerPAT)'
+  #     if ($pat -like '*PAT*' -or $pat -eq '')
+  #     {
+  #       throw 'No PAT defined'
+  #     }
+  #     $url = 'https://microsoft.artifacts.visualstudio.com/defaultcollection/_apis/symbol/symsrv'
+  #     $(symbolToolPath) --authenticated-server-path $(SymbolServerPAT) $url --symbols -d "$env:ob_outputDirectory\*" --recurse-subdirectories
+  #   displayName: 'Download Symbols for binaries'
+  #   retryCountOnTaskFailure: 2
+  #   workingDirectory: '$(repoRoot)'
 
   - pwsh: |
       Get-ChildItem '$(ob_outputDirectory)' -File -Recurse |
@@ -173,16 +187,10 @@ jobs:
           Md5Hash = (Get-FileHash -Algorithm MD5 -Path $_.FullName).Hash
           Sha512Hash = (Get-FileHash -Algorithm SHA512 -Path $_.FullName).Hash
         }
-      } | Export-Csv -Path '$(Build.SourcesDirectory)/ReleaseFileHash.csv'
+      } | Export-Csv -Path '$(Build.SourcesDirectory)\ReleaseFileHash.csv'
     workingDirectory: '$(repoRoot)'
     displayName: 'Create release file hash artifact'
 
   - pwsh: |
-      Copy-Item -Path '$(Build.SourcesDirectory)/ReleaseFileHash.csv' -Destination '$(ob_outputDirectory)' -Verbose
+      Copy-Item -Path '$(Build.SourcesDirectory)\ReleaseFileHash.csv' -Destination '$(ob_outputDirectory)' -Verbose
     displayName: 'Publish Build File Hash artifact'
-
-  - pwsh: |
-      Get-ChildItem -Path env: | Out-String -width 150 -Stream | write-Verbose -Verbose
-    displayName: Capture Environment
-    condition: succeededOrFailed()
-    workingDirectory: '$(repoRoot)'
