From 6a9c784312eff764fc981acaf6d4de39edfa976b Mon Sep 17 00:00:00 2001 From: Ciara Stacke Date: Tue, 30 Sep 2025 13:25:36 +0100 Subject: [PATCH 1/2] Optimize mesh weight conformance tests using batch requests The mesh weight conformance tests were executing 500 separate kubectl exec commands with random delays, resulting in very slow test execution. This optimization uses the echo client's --count flag to execute all 500 requests in a single batch, dramatically reducing test time. --- conformance/tests/mesh/grpcroute-weight.go | 19 +++--- conformance/tests/mesh/httproute-weight.go | 21 +++--- conformance/utils/echo/parse.go | 23 +++++++ conformance/utils/echo/pod.go | 27 ++++++++ conformance/utils/weight/senders.go | 14 ++++ conformance/utils/weight/weight.go | 74 ++++++++++++++++++++++ 6 files changed, 159 insertions(+), 19 deletions(-) diff --git a/conformance/tests/mesh/grpcroute-weight.go b/conformance/tests/mesh/grpcroute-weight.go index 0157119691..a80d1433a2 100644 --- a/conformance/tests/mesh/grpcroute-weight.go +++ b/conformance/tests/mesh/grpcroute-weight.go @@ -58,20 +58,21 @@ var MeshGRPCRouteWeight = suite.ConformanceTest{ "echo-v2": 0.3, } - sender := weight.NewFunctionBasedSender(func() (string, error) { - uniqueExpected := expected - if err := http.AddEntropy(&uniqueExpected); err != nil { - return "", fmt.Errorf("error adding entropy: %w", err) - } - _, cRes, err := client.CaptureRequestResponseAndCompare(t, uniqueExpected) + sender := weight.NewBatchFunctionBasedSender(func(count int) ([]string, error) { + responses, err := client.RequestBatch(t, expected, count) if err != nil { - return "", fmt.Errorf("failed gRPC mesh request: %w", err) + return nil, fmt.Errorf("failed batch gRPC mesh request: %w", err) + } + + hostnames := make([]string, len(responses)) + for i, resp := range responses { + hostnames[i] = resp.Hostname } - return cRes.Hostname, nil + return hostnames, nil }) for i := 0; i < weight.MaxTestRetries; i++ { - if err := weight.TestWeightedDistribution(sender, expectedWeights); err != nil { + if err := weight.TestWeightedDistributionBatch(sender, expectedWeights); err != nil { t.Logf("Traffic distribution test failed (%d/%d): %s", i+1, weight.MaxTestRetries, err) } else { return diff --git a/conformance/tests/mesh/httproute-weight.go b/conformance/tests/mesh/httproute-weight.go index ceccca95f1..ee4121a5bc 100644 --- a/conformance/tests/mesh/httproute-weight.go +++ b/conformance/tests/mesh/httproute-weight.go @@ -58,20 +58,21 @@ var MeshHTTPRouteWeight = suite.ConformanceTest{ "echo-v2": 0.3, } - sender := weight.NewFunctionBasedSender(func() (string, error) { - uniqueExpected := expected - if err := http.AddEntropy(&uniqueExpected); err != nil { - return "", fmt.Errorf("error adding entropy: %w", err) - } - _, cRes, err := client.CaptureRequestResponseAndCompare(t, uniqueExpected) + sender := weight.NewBatchFunctionBasedSender(func(count int) ([]string, error) { + responses, err := client.RequestBatch(t, expected, count) if err != nil { - return "", fmt.Errorf("failed mesh request: %w", err) + return nil, fmt.Errorf("failed batch mesh request: %w", err) + } + + hostnames := make([]string, len(responses)) + for i, resp := range responses { + hostnames[i] = resp.Hostname } - return cRes.Hostname, nil + return hostnames, nil }) - for i := 0; i < weight.MaxTestRetries; i++ { - if err := weight.TestWeightedDistribution(sender, expectedWeights); err != nil { + for i := range weight.MaxTestRetries { + if err := weight.TestWeightedDistributionBatch(sender, expectedWeights); err != nil { t.Logf("Traffic distribution test failed (%d/%d): %s", i+1, weight.MaxTestRetries, err) } else { return diff --git a/conformance/utils/echo/parse.go b/conformance/utils/echo/parse.go index edabded67b..60cb9d195f 100644 --- a/conformance/utils/echo/parse.go +++ b/conformance/utils/echo/parse.go @@ -210,6 +210,29 @@ func ParseResponse(output string) Response { return out } +// parseMultipleResponses parses output containing multiple responses separated by blank lines +func parseMultipleResponses(output string) []Response { + // Split by double newline which typically separates individual responses + // in batch mode output + responseSections := strings.Split(output, "\n\n") + + var responses []Response + for _, section := range responseSections { + section = strings.TrimSpace(section) + if section == "" { + continue + } + // Parse each section as a separate response + resp := ParseResponse(section) + // Only add responses that have meaningful content (at least a hostname or code) + if resp.Hostname != "" || resp.Code != "" { + responses = append(responses, resp) + } + } + + return responses +} + // HeaderType is a helper enum for retrieving Headers from a Response. type HeaderType string diff --git a/conformance/utils/echo/pod.go b/conformance/utils/echo/pod.go index 1a3e40fb6c..ff9ce6cb07 100644 --- a/conformance/utils/echo/pod.go +++ b/conformance/utils/echo/pod.go @@ -79,6 +79,10 @@ func (m *MeshPod) MakeRequestAndExpectEventuallyConsistentResponse(t *testing.T, } func makeRequest(t *testing.T, exp *http.ExpectedResponse) []string { + return makeRequestWithCount(t, exp, 0) +} + +func makeRequestWithCount(t *testing.T, exp *http.ExpectedResponse, count int) []string { if exp.Request.Host == "" { exp.Request.Host = "echo" } @@ -112,6 +116,9 @@ func makeRequest(t *testing.T, exp *http.ExpectedResponse) []string { for k, v := range r.Headers { args = append(args, "-H", fmt.Sprintf("%v:%v", k, v)) } + if count > 0 { + args = append(args, fmt.Sprintf("--count=%d", count)) + } return args } @@ -275,3 +282,23 @@ func (m *MeshPod) CaptureRequestResponseAndCompare(t *testing.T, exp http.Expect } return req, resp, nil } + +// RequestBatch executes a batch of requests using the --count flag and returns all responses +func (m *MeshPod) RequestBatch(t *testing.T, exp http.ExpectedResponse, count int) ([]Response, error) { + req := makeRequestWithCount(t, &exp, count) + + resp, err := m.request(req) + if err != nil { + return nil, fmt.Errorf("batch request failed: %w", err) + } + + // Split the output by response boundaries + // Each response is separated by a blank line in the output + responses := parseMultipleResponses(resp.RawContent) + + if len(responses) != count { + tlog.Logf(t, "Warning: expected %d responses but got %d", count, len(responses)) + } + + return responses, nil +} diff --git a/conformance/utils/weight/senders.go b/conformance/utils/weight/senders.go index 304ead0e9f..9dba358ad5 100644 --- a/conformance/utils/weight/senders.go +++ b/conformance/utils/weight/senders.go @@ -29,3 +29,17 @@ func (s *FunctionBasedSender) SendRequest() (string, error) { func NewFunctionBasedSender(sendFunc func() (string, error)) RequestSender { return &FunctionBasedSender{sendFunc: sendFunc} } + +// BatchFunctionBasedSender implements BatchRequestSender using a function +type BatchFunctionBasedSender struct { + sendBatchFunc func(count int) ([]string, error) +} + +func (s *BatchFunctionBasedSender) SendBatchRequest(count int) ([]string, error) { + return s.sendBatchFunc(count) +} + +// NewBatchFunctionBasedSender creates a BatchRequestSender from a function +func NewBatchFunctionBasedSender(sendBatchFunc func(count int) ([]string, error)) BatchRequestSender { + return &BatchFunctionBasedSender{sendBatchFunc: sendBatchFunc} +} diff --git a/conformance/utils/weight/weight.go b/conformance/utils/weight/weight.go index 123719ae17..12d035f9a5 100644 --- a/conformance/utils/weight/weight.go +++ b/conformance/utils/weight/weight.go @@ -55,6 +55,11 @@ func extractBackendName(podName string) string { return strings.Join(parts[:len(parts)-2], "-") } +// BatchRequestSender defines an interface for sending batch requests +type BatchRequestSender interface { + SendBatchRequest(count int) ([]string, error) +} + // TestWeightedDistribution tests that requests are distributed according to expected weights func TestWeightedDistribution(sender RequestSender, expectedWeights map[string]float64) error { const ( @@ -135,6 +140,75 @@ func TestWeightedDistribution(sender RequestSender, expectedWeights map[string]f return errors.Join(errs...) } +// TestWeightedDistributionBatch tests that requests are distributed according to expected weights +// using batch request execution for improved performance +func TestWeightedDistributionBatch(sender BatchRequestSender, expectedWeights map[string]float64) error { + const ( + tolerancePercentage = 0.05 + totalRequests = 500 + ) + + // Execute all requests in a single batch + podNames, err := sender.SendBatchRequest(totalRequests) + if err != nil { + return fmt.Errorf("error while sending batch request: %w", err) + } + + if len(podNames) != totalRequests { + return fmt.Errorf("expected %d responses but got %d", totalRequests, len(podNames)) + } + + // Count the distribution + seen := make(map[string]float64, len(expectedWeights)) + for _, podName := range podNames { + backendName := extractBackendName(podName) + + if _, exists := expectedWeights[backendName]; exists { + seen[backendName]++ + } else { + return fmt.Errorf("request was handled by an unexpected pod %q (extracted backend: %q)", podName, backendName) + } + } + + // Count how many backends should receive traffic (weight > 0) + expectedActiveBackends := 0 + for _, weight := range expectedWeights { + if weight > 0.0 { + expectedActiveBackends++ + } + } + + var errs []error + if len(seen) != expectedActiveBackends { + errs = append(errs, fmt.Errorf("expected %d backends to receive traffic, but got %d", expectedActiveBackends, len(seen))) + } + + for wantBackend, wantPercent := range expectedWeights { + gotCount, ok := seen[wantBackend] + + if !ok && wantPercent != 0.0 { + errs = append(errs, fmt.Errorf("expect traffic to hit backend %q - but none was received", wantBackend)) + continue + } + + gotPercent := gotCount / float64(totalRequests) + + if math.Abs(gotPercent-wantPercent) > tolerancePercentage { + errs = append(errs, fmt.Errorf("backend %q weighted traffic of %v not within tolerance %v (+/-%f)", + wantBackend, + gotPercent, + wantPercent, + tolerancePercentage, + )) + } + } + + slices.SortFunc(errs, func(a, b error) int { + return cmp.Compare(a.Error(), b.Error()) + }) + return errors.Join(errs...) +} + // Entropy utilities // addRandomDelay adds a random delay up to the specified limit in milliseconds From 7cf0f131bf8d95ae131f09c860732bd179b0fd8a Mon Sep 17 00:00:00 2001 From: Ciara Stacke Date: Thu, 2 Oct 2025 11:30:24 +0100 Subject: [PATCH 2/2] Wrong count warning should be an error --- conformance/utils/echo/pod.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conformance/utils/echo/pod.go b/conformance/utils/echo/pod.go index ff9ce6cb07..38ab42048c 100644 --- a/conformance/utils/echo/pod.go +++ b/conformance/utils/echo/pod.go @@ -297,7 +297,7 @@ func (m *MeshPod) RequestBatch(t *testing.T, exp http.ExpectedResponse, count in responses := parseMultipleResponses(resp.RawContent) if len(responses) != count { - tlog.Logf(t, "Warning: expected %d responses but got %d", count, len(responses)) + return nil, fmt.Errorf("expected %d responses but got %d", count, len(responses)) } return responses, nil