build: implement performance tracking workflow (#443)

Author: oameye

.github/workflows/Benchmarks.yml

@@ -0,0 +1,69 @@
+name: Benchmark Tracking
+on:
+  push:
+    branches:
+      - 'master'
+    paths:
+      - '.github/workflows/Benchmarks.yml'
+      - 'src/**'
+      - 'ext/**'
+      - 'test/runtests.jl'
+      - 'test/core-test/**'
+      - 'test/ext-test/cpu/**'
+      - 'Project.toml'
+  pull_request:
+    branches:
+      - 'master'
+    paths:
+      - '.github/workflows/Benchmarks.yml'
+      - 'src/**'
+      - 'ext/**'
+      - 'test/**'
+      - 'Project.toml'
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
+  workflow_dispatch:
+
+permissions:
+    actions: write
+    contents: write
+    deployments: write
+
+jobs:
+  benchmark:
+    runs-on: ubuntu-latest
+    if: ${{ !github.event.pull_request.draft }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: julia-actions/setup-julia@v2
+        with:
+          version: '1'
+          arch: x64
+      - uses: julia-actions/cache@v2
+      - name: Run benchmark
+        run: |
+          cd benchmarks
+          julia --project --threads=2 --color=yes -e '
+            using Pkg;
+            Pkg.develop(PackageSpec(path=joinpath(pwd(), "..")));
+            Pkg.instantiate();
+            include("runbenchmarks.jl")'
+
+      # this will update benchmarks/data.js in gh-pages branch
+      - name: Parse & Upload Benchmark Results
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: Benchmark Results
+          tool: "julia"
+          output-file-path: benchmarks/benchmarks_output.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          alert-threshold: "130%"
+          fail-threshold: "170%"
+          comment-on-alert: true
+          fail-on-alert: true
+          benchmark-data-dir-path: benchmarks
+          max-items-in-chart: 100
+          auto-push: ${{ github.event_name != 'pull_request' }}

.github/workflows/downgrade.yml

@@ -0,0 +1,120 @@
+using HarmonicBalance, HarmonicSteadyState
+using HarmonicSteadyState: HomotopyContinuationProblem, OrderedDict
+using HarmonicSteadyState: sort_solutions, _classify_default!
+
+function benchmark_kpo!(SUITE)
+    @variables ω₀ γ λ F η α ω t x(t)
+
+    natural_equation =
+        d(d(x, t), t) +
+        γ * d(x, t) +
+        (ω₀^2 - λ * cos(2 * ω * t)) * x +
+        α * x^3 +
+        η * d(x, t) * x^2
+    forces = F * cos(ω * t)
+    diff_eq = DifferentialEquation(natural_equation + forces, x)
+    add_harmonic!(diff_eq, x, ω)
+
+    harmonic_eq = get_harmonic_equations(diff_eq)
+
+    SUITE["Construction"]["Harmonic Equation"]["One Frequency"] = @benchmarkable begin
+        get_harmonic_equations($diff_eq)
+    end seconds = 10
+
+    krylov_eq = get_krylov_equations(diff_eq; order=1)
+    krylov_eq2 = get_krylov_equations(diff_eq; order=2)
+
+    SUITE["Construction"]["Krylov Equation"]["Order 1"] = @benchmarkable begin
+        get_krylov_equations($diff_eq; order=1)
+    end seconds = 10
+    SUITE["Construction"]["Krylov Equation"]["Order 2"] = @benchmarkable begin
+        get_krylov_equations($diff_eq; order=2)
+    end seconds = 10
+
+    fixed = OrderedDict(ω₀ => 1.0, γ => 1e-2, λ => 5e-2, F => 1e-3, α => 1.0, η => 0.3)
+    varied = OrderedDict(ω => range(0.9, 1.1, 100))
+
+    problem = HomotopyContinuationProblem(harmonic_eq, varied, fixed)
+    SUITE["Construction"]["Problem"]["HomotopyContinuationProblem"] = @benchmarkable begin
+        HomotopyContinuationProblem($harmonic_eq, $varied, $fixed)
+    end seconds = 10
+
+    show_progress = false
+    sorting = "no_sorting"
+    classify_default = false
+
+    method = WarmUp(; thread=false)
+    result = get_steady_states(problem, method; show_progress, sorting, classify_default)
+
+    SUITE["Steady states"]["Homotopy Problem"]["Warm up method"] = @benchmarkable begin
+        get_steady_states(
+            $problem,
+            $method;
+            show_progress=false,
+            sorting="no_sorting",
+            classify_default=false,
+        )
+    end seconds = 10
+
+    method = TotalDegree(; thread=false)
+    result = get_steady_states(problem, method; show_progress, sorting, classify_default)
+
+    SUITE["Steady states"]["Homotopy Problem"]["Total degree homotopy"] = @benchmarkable begin
+        get_steady_states(
+            $problem,
+            $method;
+            show_progress=false,
+            sorting="no_sorting",
+            classify_default=false,
+        )
+    end seconds = 10
+
+    method = Polyhedral(; thread=false)
+    result = get_steady_states(problem, method; show_progress, sorting, classify_default)
+
+    SUITE["Steady states"]["Homotopy Problem"]["Polyhedral homotopy"] = @benchmarkable begin
+        get_steady_states(
+            $problem,
+            $method;
+            show_progress=false,
+            sorting="no_sorting",
+            classify_default=false,
+        )
+    end seconds = 10
+
+    solutions_not_sorted = result.solutions
+    sort_solutions(solutions_not_sorted; sorting="nearest", show_progress=false)
+    sort_solutions(solutions_not_sorted; sorting="hilbert", show_progress=false)
+
+    SUITE["Sorting"]["One dimensional"]["Nearest-neighbor sorting"] = @benchmarkable begin
+        sort_solutions($solutions_not_sorted; show_progress=false, sorting="nearest")
+    end seconds = 10
+
+    SUITE["Sorting"]["One dimensional"]["Hilbert sorting"] = @benchmarkable begin
+        sort_solutions($solutions_not_sorted; show_progress=false, sorting="hilbert")
+    end seconds = 10
+
+    _classify_default!(deepcopy(result))
+    SUITE["Classification"]["One Dimensional"]["Default classifications"] = @benchmarkable begin
+        _classify_default!(x)
+    end setup = (x = deepcopy($result)) evals = 1 seconds = 10
+
+    method = WarmUp(; thread=false)
+    result = get_steady_states(problem, method; show_progress)
+
+    Ω_range = range(0.95, 1.05, 100)
+    get_jacobian_response(result, x, Ω_range, 3; show_progress=false)
+    get_rotframe_jacobian_response(result, Ω_range, 3; show_progress=false, damping_mod=1.0)
+
+    SUITE["Linear response"]["Lab frame"]["Jacobian Response"] = @benchmarkable begin
+        get_jacobian_response($result, $x, $(Ω_range), 3; show_progress=false)
+    end seconds = 10
+
+    SUITE["Linear response"]["Rotating frame"]["Jacobian Response"] = @benchmarkable begin
+        get_rotframe_jacobian_response(
+            $result, $(Ω_range), 3; show_progress=false, damping_mod=1.0
+        )
+    end seconds = 10
+
+    return nothing
+end

benchmarks/KPO.jl

@@ -0,0 +1,16 @@
+[deps]
+BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
+DocumenterTools = "35a29f4d-8980-5a13-9543-d66fff28ecb8"
+HarmonicBalance = "e13b9ff6-59c3-11ec-14b1-f3d2cc6c135e"
+HarmonicSteadyState = "1158f75c-a779-4b85-8bfb-8fcf6bf02ced"
+OrdinaryDiffEq = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed"
+OrdinaryDiffEqRosenbrock = "43230ef6-c299-4910-a778-202eb28ce4ce"
+OrdinaryDiffEqTsit5 = "b1df2697-797e-41e3-8120-5422d3b24e4a"
+Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
+QuantumCumulants = "35bcea6d-e19f-57db-af74-8011de6c7255"
+SteadyStateDiffEq = "9672c7b4-1e72-59bd-8a11-6ac3964bc41f"
+
+[compat]
+Documenter = "1"
+julia = "1.10"

benchmarks/Project.toml

@@ -0,0 +1,14 @@
+using BenchmarkTools
+using HarmonicBalance, HarmonicSteadyState, QuantumCumulants
+
+const SUITE = BenchmarkGroup()
+
+include("KPO.jl")
+
+benchmark_kpo!(SUITE)
+
+BenchmarkTools.tune!(SUITE)
+results = BenchmarkTools.run(SUITE; verbose=true)
+display(median(results))
+
+BenchmarkTools.save("benchmarks_output.json", median(results))