From 446ca0b79ee4187140008cc9e22efb9ef83ad39d Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Fri, 7 Nov 2025 20:49:42 +0000 Subject: [PATCH 01/65] Test changes to add leak detection --- .../CosmosNettyLeakDetectorFactory.java | 128 ++++++++++++++++++ .../cosmos/encryption/TestSuiteBase.java | 58 ++++++++ .../azure/cosmos/CosmosAsyncClientTest.java | 1 + .../CosmosNettyLeakDetectorFactory.java | 128 ++++++++++++++++++ .../cosmos/implementation/TestSuiteBase.java | 61 ++++++++- .../com/azure/cosmos/rx/TestSuiteBase.java | 62 +++++++++ .../src/test/resources/cfp-split-testng.xml | 3 + .../circuit-breaker-misc-direct-testng.xml | 3 + .../circuit-breaker-misc-gateway-testng.xml | 3 + ...cuit-breaker-read-all-read-many-testng.xml | 3 + .../src/test/resources/direct-testng.xml | 3 + .../src/test/resources/e2e-testng.xml | 3 + .../src/test/resources/emulator-testng.xml | 3 + .../test/resources/emulator-vnext-testng.xml | 3 + .../src/test/resources/examples-testng.xml | 3 + .../src/test/resources/fast-testng.xml | 3 + .../test/resources/fi-multi-master-testng.xml | 3 + .../fi-thinclient-multi-master-testng.xml | 3 + .../fi-thinclient-multi-region-testng.xml | 3 + .../resources/flaky-multi-master-testng.xml | 3 + .../test/resources/long-emulator-testng.xml | 3 + .../src/test/resources/long-testng.xml | 3 + .../test/resources/multi-master-testng.xml | 3 + .../test/resources/multi-region-testng.xml | 3 + .../src/test/resources/query-testng.xml | 3 + .../src/test/resources/split-testng.xml | 3 + .../src/test/resources/thinclient-testng.xml | 3 + .../src/test/resources/unit-testng.xml | 3 + .../implementation/AsyncDocumentClient.java | 9 +- .../azure/cosmos/implementation/Configs.java | 12 ++ .../implementation/RxDocumentClientImpl.java | 35 +++++ .../cosmos/implementation/StackTraceUtil.java | 46 +++++++ 32 files changed, 604 insertions(+), 2 deletions(-) create mode 100644 sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java create mode 100644 sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java create mode 100644 sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/StackTraceUtil.java diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java new file mode 100644 index 000000000000..c4a602cedaff --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java @@ -0,0 +1,128 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +package com.azure.cosmos.encryption; + +import com.azure.cosmos.implementation.StackTraceUtil; +import io.netty.util.ResourceLeakDetector; +import io.netty.util.ResourceLeakDetectorFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.IExecutionListener; + +import java.util.ArrayList; +import java.util.List; + +public final class CosmosNettyLeakDetectorFactory extends ResourceLeakDetectorFactory implements IExecutionListener { + protected static Logger logger = LoggerFactory.getLogger(CosmosNettyLeakDetectorFactory.class.getSimpleName()); + private final static List identifiedLeaks = new ArrayList<>(); + private final static Object staticLock = new Object(); + private static volatile boolean isLeakDetectionDisabled = false; + private static volatile boolean isInitialized = false; + + private CosmosNettyLeakDetectorFactory() { + } + + @Override + public void onExecutionStart() { + ingestIntoNetty(); + } + + @Override + public void onExecutionFinish() { + // Run GC to force finalizers to run - only in finalizers Netty would actually detect any leaks. + System.gc(); + } + + // This method must be called as early as possible in the lifecycle of a process + // before any Netty ByteBNuf has been allocated + public static void ingestIntoNetty() { + if (isInitialized) { + return; + } + + synchronized (staticLock) { + if (isInitialized) { + return; + } + + // Must run before any Netty ByteBuf is allocated + ResourceLeakDetector.setLevel(ResourceLeakDetector.Level.PARANOID); + // sample every allocation + System.setProperty("io.netty.leakDetection.samplingInterval", "1"); + // install custom reporter + ResourceLeakDetectorFactory.setResourceLeakDetectorFactory(new CosmosNettyLeakDetectorFactory()); + + isInitialized = true; + } + } + + public static List resetIdentifiedLeaks() { + // Run GC to force finalizers to run - only in finalizers Netty would actually detect any leaks. + System.gc(); + synchronized (staticLock) { + List leaksSnapshot = new ArrayList<>(identifiedLeaks); + + identifiedLeaks.clear(); + return leaksSnapshot; + } + } + + public static AutoCloseable createDisableLeakDetectionScope() { + synchronized (staticLock) { + logger.info("Disabling Leak detection: {}", StackTraceUtil.currentCallStack()); + return new DisableLeakDetectionScope(); + } + } + + @Override + public ResourceLeakDetector newResourceLeakDetector( + Class resource, int samplingInterval, long maxActive) { + + return new ResourceLeakDetector(resource, samplingInterval, maxActive) { + @Override + protected void reportTracedLeak(String resourceType, String records) { + synchronized (staticLock) { + if (!isLeakDetectionDisabled) { + String msg = "NETTY LEAK (traced) type=" + + resourceType + + "records=\n" + + records; + + identifiedLeaks.add(msg); + logger.error(msg); + } + } + } + + @Override + protected void reportUntracedLeak(String resourceType) { + synchronized (staticLock) { + String msg = "NETTY LEAK (untraced) type=" + resourceType; + + identifiedLeaks.add(msg); + logger.error(msg); + } + } + + @Override + protected void reportInstancesLeak(String resourceType) { + synchronized (staticLock) { + String msg = "NETTY LEAK (instances) type=" + resourceType; + + identifiedLeaks.add(msg); + logger.error(msg); + } + } + }; + } + + private static final class DisableLeakDetectionScope implements AutoCloseable { + @Override + public void close() { + synchronized (staticLock) { + isLeakDetectionDisabled = false; + logger.info("Leak detection enabled again."); + } + } + } +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java index c44b96f5ab25..d121c33595c3 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java @@ -26,6 +26,7 @@ import com.azure.cosmos.implementation.ConnectionPolicy; import com.azure.cosmos.implementation.InternalObjectNode; import com.azure.cosmos.implementation.PathParser; +import com.azure.cosmos.implementation.RxDocumentClientImpl; import com.azure.cosmos.implementation.TestConfigurations; import com.azure.cosmos.implementation.Utils; import com.azure.cosmos.implementation.directconnectivity.Protocol; @@ -62,7 +63,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.ITestContext; +import org.testng.annotations.AfterClass; import org.testng.annotations.AfterSuite; +import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeSuite; import org.testng.annotations.DataProvider; import org.testng.annotations.Listeners; @@ -73,7 +76,9 @@ import java.time.Duration; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -81,6 +86,7 @@ import static com.azure.cosmos.BridgeInternal.extractConfigs; import static com.azure.cosmos.BridgeInternal.injectConfigs; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Fail.fail; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.spy; @@ -110,6 +116,8 @@ public class TestSuiteBase extends CosmosEncryptionAsyncClientTest { protected int subscriberValidationTimeout = TIMEOUT; + private volatile Map activeClientsAtBegin = new HashMap<>(); + private static CosmosAsyncDatabase SHARED_DATABASE; private static CosmosAsyncContainer SHARED_MULTI_PARTITION_COLLECTION_WITH_ID_AS_PARTITION_KEY; private static CosmosAsyncContainer SHARED_MULTI_PARTITION_COLLECTION; @@ -226,6 +234,56 @@ public CosmosAsyncDatabase getDatabase(String id) { } } + @BeforeClass(groups = {"fast", "long", "direct", "multi-master", "encryption"}, timeOut = SUITE_SETUP_TIMEOUT) + + public void beforeClassSetupLeakDetection() { + CosmosNettyLeakDetectorFactory.ingestIntoNetty(); + this.activeClientsAtBegin = RxDocumentClientImpl.getActiveClientsSnapshot(); + } + + @AfterClass(groups = {"fast", "long", "direct", "multi-master", "encryption"}, timeOut = SUITE_SETUP_TIMEOUT) + public void afterClassSetupLeakDetection() { + + Map leakedClientSnapshotNow = RxDocumentClientImpl.getActiveClientsSnapshot(); + StringBuilder sb = new StringBuilder(); + Map leakedClientSnapshotAtBegin = activeClientsAtBegin; + + for (Integer clientId : leakedClientSnapshotNow.keySet()) { + if (!leakedClientSnapshotAtBegin.containsKey(clientId)) { + // this client was leaked in this class + sb + .append("CosmosClient [") + .append(clientId) + .append("] leaked. Callstack of initialization:\n") + .append(leakedClientSnapshotNow.get(clientId)); + } + } + + if (sb.length() > 0) { + String msg = "\"COSMOS CLIENT LEAKS detected in test class: " + + this.getClass().getCanonicalName() + + sb; + + logger.error(msg); + // fail(msg); + } + + List nettyLeaks = CosmosNettyLeakDetectorFactory.resetIdentifiedLeaks(); + if (nettyLeaks.size() > 0) { + sb.append("\n"); + for (String leak : nettyLeaks) { + sb.append(leak).append("\n"); + } + + String msg = "\"NETTY LEAKS detected in test class: " + + this.getClass().getCanonicalName() + + sb; + + logger.error(msg); + // fail(msg); + } + } + @BeforeSuite(groups = {"fast", "long", "direct", "multi-master", "encryption"}, timeOut = SUITE_SETUP_TIMEOUT) public static void beforeSuite() { diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosAsyncClientTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosAsyncClientTest.java index 1324649ecdf3..a1bfb4abf41b 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosAsyncClientTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosAsyncClientTest.java @@ -10,6 +10,7 @@ import com.azure.cosmos.models.PartitionKey; import org.testng.ITest; import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; import java.lang.reflect.Method; diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java new file mode 100644 index 000000000000..a889d700c555 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java @@ -0,0 +1,128 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +package com.azure.cosmos; + +import com.azure.cosmos.implementation.StackTraceUtil; +import io.netty.util.ResourceLeakDetector; +import io.netty.util.ResourceLeakDetectorFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.IExecutionListener; + +import java.util.ArrayList; +import java.util.List; + +public final class CosmosNettyLeakDetectorFactory extends ResourceLeakDetectorFactory implements IExecutionListener { + protected static Logger logger = LoggerFactory.getLogger(CosmosNettyLeakDetectorFactory.class.getSimpleName()); + private final static List identifiedLeaks = new ArrayList<>(); + private final static Object staticLock = new Object(); + private static volatile boolean isLeakDetectionDisabled = false; + private static volatile boolean isInitialized = false; + + private CosmosNettyLeakDetectorFactory() { + } + + @Override + public void onExecutionStart() { + ingestIntoNetty(); + } + + @Override + public void onExecutionFinish() { + // Run GC to force finalizers to run - only in finalizers Netty would actually detect any leaks. + System.gc(); + } + + // This method must be called as early as possible in the lifecycle of a process + // before any Netty ByteBNuf has been allocated + public static void ingestIntoNetty() { + if (isInitialized) { + return; + } + + synchronized (staticLock) { + if (isInitialized) { + return; + } + + // Must run before any Netty ByteBuf is allocated + ResourceLeakDetector.setLevel(ResourceLeakDetector.Level.PARANOID); + // sample every allocation + System.setProperty("io.netty.leakDetection.samplingInterval", "1"); + // install custom reporter + ResourceLeakDetectorFactory.setResourceLeakDetectorFactory(new CosmosNettyLeakDetectorFactory()); + + isInitialized = true; + } + } + + public static List resetIdentifiedLeaks() { + // Run GC to force finalizers to run - only in finalizers Netty would actually detect any leaks. + System.gc(); + synchronized (staticLock) { + List leaksSnapshot = new ArrayList<>(identifiedLeaks); + + identifiedLeaks.clear(); + return leaksSnapshot; + } + } + + public static AutoCloseable createDisableLeakDetectionScope() { + synchronized (staticLock) { + logger.info("Disabling Leak detection: {}", StackTraceUtil.currentCallStack()); + return new DisableLeakDetectionScope(); + } + } + + @Override + public ResourceLeakDetector newResourceLeakDetector( + Class resource, int samplingInterval, long maxActive) { + + return new ResourceLeakDetector(resource, samplingInterval, maxActive) { + @Override + protected void reportTracedLeak(String resourceType, String records) { + synchronized (staticLock) { + if (!isLeakDetectionDisabled) { + String msg = "NETTY LEAK (traced) type=" + + resourceType + + "records=\n" + + records; + + identifiedLeaks.add(msg); + logger.error(msg); + } + } + } + + @Override + protected void reportUntracedLeak(String resourceType) { + synchronized (staticLock) { + String msg = "NETTY LEAK (untraced) type=" + resourceType; + + identifiedLeaks.add(msg); + logger.error(msg); + } + } + + @Override + protected void reportInstancesLeak(String resourceType) { + synchronized (staticLock) { + String msg = "NETTY LEAK (instances) type=" + resourceType; + + identifiedLeaks.add(msg); + logger.error(msg); + } + } + }; + } + + private static final class DisableLeakDetectionScope implements AutoCloseable { + @Override + public void close() { + synchronized (staticLock) { + isLeakDetectionDisabled = false; + logger.info("Leak detection enabled again."); + } + } + } +} diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestSuiteBase.java index 19e6bc24266e..19f3c1552316 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestSuiteBase.java @@ -5,6 +5,7 @@ import com.azure.cosmos.ConsistencyLevel; import com.azure.cosmos.CosmosAsyncClient; import com.azure.cosmos.CosmosClientBuilder; +import com.azure.cosmos.CosmosNettyLeakDetectorFactory; import com.azure.cosmos.DirectConnectionConfig; import com.azure.cosmos.DocumentClientTest; import com.azure.cosmos.GatewayConnectionConfig; @@ -38,7 +39,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.ITestContext; +import org.testng.annotations.AfterClass; import org.testng.annotations.AfterSuite; +import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeSuite; import org.testng.annotations.DataProvider; import org.testng.annotations.Listeners; @@ -48,17 +51,19 @@ import java.time.Duration; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Fail.fail; import static org.mockito.Mockito.doAnswer; @Listeners({TestNGLogListener.class}) public class TestSuiteBase extends DocumentClientTest { - private static final int DEFAULT_BULK_INSERT_CONCURRENCY_LEVEL = 500; private static final ObjectMapper objectMapper = new ObjectMapper(); protected static Logger logger = LoggerFactory.getLogger(TestSuiteBase.class.getSimpleName()); @@ -83,6 +88,8 @@ public class TestSuiteBase extends DocumentClientTest { protected static DocumentCollection SHARED_SINGLE_PARTITION_COLLECTION; protected static DocumentCollection SHARED_MULTI_PARTITION_COLLECTION_WITH_COMPOSITE_AND_SPATIAL_INDEXES; + private volatile Map activeClientsAtBegin = new HashMap<>(); + private static ImmutableList immutableListOrNull(List list) { return list != null ? ImmutableList.copyOf(list) : null; } @@ -143,6 +150,58 @@ public Mono> deleteDatabase(String id) { } } + @BeforeClass(groups = {"fast", "long", "direct", "multi-region", "multi-master", "flaky-multi-master", "emulator", + "split", "query", "cfp-split", "long-emulator"}, timeOut = SUITE_SETUP_TIMEOUT) + + public void beforeClassSetupLeakDetection() { + CosmosNettyLeakDetectorFactory.ingestIntoNetty(); + this.activeClientsAtBegin = RxDocumentClientImpl.getActiveClientsSnapshot(); + } + + @AfterClass(groups = {"fast", "long", "direct", "multi-region", "multi-master", "flaky-multi-master", "emulator", + "split", "query", "cfp-split", "long-emulator"}, timeOut = SUITE_SETUP_TIMEOUT) + public void afterClassSetupLeakDetection() { + + Map leakedClientSnapshotNow = RxDocumentClientImpl.getActiveClientsSnapshot(); + StringBuilder sb = new StringBuilder(); + Map leakedClientSnapshotAtBegin = activeClientsAtBegin; + + for (Integer clientId : leakedClientSnapshotNow.keySet()) { + if (!leakedClientSnapshotAtBegin.containsKey(clientId)) { + // this client was leaked in this class + sb + .append("CosmosClient [") + .append(clientId) + .append("] leaked. Callstack of initialization:\n") + .append(leakedClientSnapshotNow.get(clientId)); + } + } + + if (sb.length() > 0) { + String msg = "\"COSMOS CLIENT LEAKS detected in test class: " + + this.getClass().getCanonicalName() + + sb; + + logger.error(msg); + // fail(msg); + } + + List nettyLeaks = CosmosNettyLeakDetectorFactory.resetIdentifiedLeaks(); + if (nettyLeaks.size() > 0) { + sb.append("\n"); + for (String leak : nettyLeaks) { + sb.append(leak).append("\n"); + } + + String msg = "\"NETTY LEAKS detected in test class: " + + this.getClass().getCanonicalName() + + sb; + + logger.error(msg); + // fail(msg); + } + } + @BeforeSuite(groups = {"fast", "long", "direct", "multi-region", "multi-master", "flaky-multi-master", "emulator", "split", "query", "cfp-split", "long-emulator"}, timeOut = SUITE_SETUP_TIMEOUT) public static void beforeSuite() { diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java index 45efee571ed1..bbc4241d9d95 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java @@ -17,6 +17,7 @@ import com.azure.cosmos.CosmosDatabaseForTest; import com.azure.cosmos.CosmosEndToEndOperationLatencyPolicyConfigBuilder; import com.azure.cosmos.CosmosException; +import com.azure.cosmos.CosmosNettyLeakDetectorFactory; import com.azure.cosmos.CosmosResponseValidator; import com.azure.cosmos.DirectConnectionConfig; import com.azure.cosmos.GatewayConnectionConfig; @@ -32,6 +33,7 @@ import com.azure.cosmos.implementation.InternalObjectNode; import com.azure.cosmos.implementation.PathParser; import com.azure.cosmos.implementation.Resource; +import com.azure.cosmos.implementation.RxDocumentClientImpl; import com.azure.cosmos.implementation.TestConfigurations; import com.azure.cosmos.implementation.Utils; import com.azure.cosmos.implementation.directconnectivity.Protocol; @@ -72,7 +74,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.ITestContext; +import org.testng.annotations.AfterClass; import org.testng.annotations.AfterSuite; +import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeSuite; import org.testng.annotations.DataProvider; import org.testng.annotations.Listeners; @@ -85,7 +89,9 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -93,6 +99,7 @@ import static com.azure.cosmos.BridgeInternal.extractConfigs; import static com.azure.cosmos.BridgeInternal.injectConfigs; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Fail.fail; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.spy; @@ -122,6 +129,8 @@ public class TestSuiteBase extends CosmosAsyncClientTest { protected int subscriberValidationTimeout = TIMEOUT; + private volatile Map activeClientsAtBegin = new HashMap<>(); + private static CosmosAsyncDatabase SHARED_DATABASE; private static CosmosAsyncContainer SHARED_MULTI_PARTITION_COLLECTION_WITH_ID_AS_PARTITION_KEY; private static CosmosAsyncContainer SHARED_MULTI_PARTITION_COLLECTION; @@ -205,6 +214,59 @@ public CosmosAsyncDatabase getDatabase(String id) { } } + @BeforeClass(groups = {"thinclient", "fast", "long", "direct", "multi-region", "multi-master", "flaky-multi-master", "emulator", + "emulator-vnext", "split", "query", "cfp-split", "circuit-breaker-misc-gateway", "circuit-breaker-misc-direct", + "circuit-breaker-read-all-read-many", "fi-multi-master", "long-emulator", "fi-thinclient-multi-region", "fi-thinclient-multi-master"}, timeOut = SUITE_SETUP_TIMEOUT) + public void beforeClassSetupLeakDetection() { + CosmosNettyLeakDetectorFactory.ingestIntoNetty(); + this.activeClientsAtBegin = RxDocumentClientImpl.getActiveClientsSnapshot(); + } + + @AfterClass(groups = {"thinclient", "fast", "long", "direct", "multi-region", "multi-master", "flaky-multi-master", "emulator", + "emulator-vnext", "split", "query", "cfp-split", "circuit-breaker-misc-gateway", "circuit-breaker-misc-direct", + "circuit-breaker-read-all-read-many", "fi-multi-master", "long-emulator", "fi-thinclient-multi-region", "fi-thinclient-multi-master"}, timeOut = SUITE_SETUP_TIMEOUT) + public void afterClassSetupLeakDetection() { + + Map leakedClientSnapshotNow = RxDocumentClientImpl.getActiveClientsSnapshot(); + StringBuilder sb = new StringBuilder(); + Map leakedClientSnapshotAtBegin = activeClientsAtBegin; + + for (Integer clientId : leakedClientSnapshotNow.keySet()) { + if (!leakedClientSnapshotAtBegin.containsKey(clientId)) { + // this client was leaked in this class + sb + .append("CosmosClient [") + .append(clientId) + .append("] leaked. Callstack of initialization:\n") + .append(leakedClientSnapshotNow.get(clientId)); + } + } + + if (sb.length() > 0) { + String msg = "\"COSMOS CLIENT LEAKS detected in test class: " + + this.getClass().getCanonicalName() + + sb; + + logger.error(msg); + // fail(msg); + } + + List nettyLeaks = CosmosNettyLeakDetectorFactory.resetIdentifiedLeaks(); + if (nettyLeaks.size() > 0) { + sb.append("\n"); + for (String leak : nettyLeaks) { + sb.append(leak).append("\n"); + } + + String msg = "\"NETTY LEAKS detected in test class: " + + this.getClass().getCanonicalName() + + sb; + + logger.error(msg); + // fail(msg); + } + } + @BeforeSuite(groups = {"thinclient", "fast", "long", "direct", "multi-region", "multi-master", "flaky-multi-master", "emulator", "emulator-vnext", "split", "query", "cfp-split", "circuit-breaker-misc-gateway", "circuit-breaker-misc-direct", "circuit-breaker-read-all-read-many", "fi-multi-master", "long-emulator", "fi-thinclient-multi-region", "fi-thinclient-multi-master"}, timeOut = SUITE_SETUP_TIMEOUT) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/resources/cfp-split-testng.xml b/sdk/cosmos/azure-cosmos-tests/src/test/resources/cfp-split-testng.xml index bbdb7043668a..d39a972d2aec 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/resources/cfp-split-testng.xml +++ b/sdk/cosmos/azure-cosmos-tests/src/test/resources/cfp-split-testng.xml @@ -22,6 +22,9 @@ --> + + + diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/resources/circuit-breaker-misc-direct-testng.xml b/sdk/cosmos/azure-cosmos-tests/src/test/resources/circuit-breaker-misc-direct-testng.xml index 37adba46a374..2ac6db27163d 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/resources/circuit-breaker-misc-direct-testng.xml +++ b/sdk/cosmos/azure-cosmos-tests/src/test/resources/circuit-breaker-misc-direct-testng.xml @@ -22,6 +22,9 @@ --> + + + diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/resources/circuit-breaker-misc-gateway-testng.xml b/sdk/cosmos/azure-cosmos-tests/src/test/resources/circuit-breaker-misc-gateway-testng.xml index b68cad70628e..abdf3c0e1e22 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/resources/circuit-breaker-misc-gateway-testng.xml +++ b/sdk/cosmos/azure-cosmos-tests/src/test/resources/circuit-breaker-misc-gateway-testng.xml @@ -22,6 +22,9 @@ --> + + + diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/resources/circuit-breaker-read-all-read-many-testng.xml b/sdk/cosmos/azure-cosmos-tests/src/test/resources/circuit-breaker-read-all-read-many-testng.xml index 541cd8cbb867..1fccc58fc151 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/resources/circuit-breaker-read-all-read-many-testng.xml +++ b/sdk/cosmos/azure-cosmos-tests/src/test/resources/circuit-breaker-read-all-read-many-testng.xml @@ -22,6 +22,9 @@ --> + + + diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/resources/direct-testng.xml b/sdk/cosmos/azure-cosmos-tests/src/test/resources/direct-testng.xml index 293d3030e9e9..64951b687986 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/resources/direct-testng.xml +++ b/sdk/cosmos/azure-cosmos-tests/src/test/resources/direct-testng.xml @@ -22,6 +22,9 @@ --> + + + diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/resources/e2e-testng.xml b/sdk/cosmos/azure-cosmos-tests/src/test/resources/e2e-testng.xml index 17b22dc377f5..98325c1bd9ae 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/resources/e2e-testng.xml +++ b/sdk/cosmos/azure-cosmos-tests/src/test/resources/e2e-testng.xml @@ -22,6 +22,9 @@ --> + + + diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/resources/emulator-testng.xml b/sdk/cosmos/azure-cosmos-tests/src/test/resources/emulator-testng.xml index 59d8973d566c..066030582262 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/resources/emulator-testng.xml +++ b/sdk/cosmos/azure-cosmos-tests/src/test/resources/emulator-testng.xml @@ -22,6 +22,9 @@ --> + + + diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/resources/emulator-vnext-testng.xml b/sdk/cosmos/azure-cosmos-tests/src/test/resources/emulator-vnext-testng.xml index 11b2376efbac..8b0ed83728e9 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/resources/emulator-vnext-testng.xml +++ b/sdk/cosmos/azure-cosmos-tests/src/test/resources/emulator-vnext-testng.xml @@ -22,6 +22,9 @@ --> + + + diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/resources/examples-testng.xml b/sdk/cosmos/azure-cosmos-tests/src/test/resources/examples-testng.xml index cc10ecf164d9..0dbd85186569 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/resources/examples-testng.xml +++ b/sdk/cosmos/azure-cosmos-tests/src/test/resources/examples-testng.xml @@ -22,6 +22,9 @@ --> + + + diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/resources/fast-testng.xml b/sdk/cosmos/azure-cosmos-tests/src/test/resources/fast-testng.xml index 3e241e961493..684fd2e99dad 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/resources/fast-testng.xml +++ b/sdk/cosmos/azure-cosmos-tests/src/test/resources/fast-testng.xml @@ -22,6 +22,9 @@ --> + + + diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/resources/fi-multi-master-testng.xml b/sdk/cosmos/azure-cosmos-tests/src/test/resources/fi-multi-master-testng.xml index 0d2077507e3a..95720e19ec27 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/resources/fi-multi-master-testng.xml +++ b/sdk/cosmos/azure-cosmos-tests/src/test/resources/fi-multi-master-testng.xml @@ -22,6 +22,9 @@ --> + + + diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/resources/fi-thinclient-multi-master-testng.xml b/sdk/cosmos/azure-cosmos-tests/src/test/resources/fi-thinclient-multi-master-testng.xml index 9bec50144d72..3f5cc579a414 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/resources/fi-thinclient-multi-master-testng.xml +++ b/sdk/cosmos/azure-cosmos-tests/src/test/resources/fi-thinclient-multi-master-testng.xml @@ -22,6 +22,9 @@ --> + + + diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/resources/fi-thinclient-multi-region-testng.xml b/sdk/cosmos/azure-cosmos-tests/src/test/resources/fi-thinclient-multi-region-testng.xml index b6e6cb8c3333..c175292f121e 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/resources/fi-thinclient-multi-region-testng.xml +++ b/sdk/cosmos/azure-cosmos-tests/src/test/resources/fi-thinclient-multi-region-testng.xml @@ -22,6 +22,9 @@ --> + + + diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/resources/flaky-multi-master-testng.xml b/sdk/cosmos/azure-cosmos-tests/src/test/resources/flaky-multi-master-testng.xml index 9e246f77704b..174bfcec036f 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/resources/flaky-multi-master-testng.xml +++ b/sdk/cosmos/azure-cosmos-tests/src/test/resources/flaky-multi-master-testng.xml @@ -22,6 +22,9 @@ --> + + + diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/resources/long-emulator-testng.xml b/sdk/cosmos/azure-cosmos-tests/src/test/resources/long-emulator-testng.xml index 24bc9fa4bd5c..9359b0a8abbb 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/resources/long-emulator-testng.xml +++ b/sdk/cosmos/azure-cosmos-tests/src/test/resources/long-emulator-testng.xml @@ -22,6 +22,9 @@ --> + + + diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/resources/long-testng.xml b/sdk/cosmos/azure-cosmos-tests/src/test/resources/long-testng.xml index d19f387a130b..d9b096c96eae 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/resources/long-testng.xml +++ b/sdk/cosmos/azure-cosmos-tests/src/test/resources/long-testng.xml @@ -22,6 +22,9 @@ --> + + + diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/resources/multi-master-testng.xml b/sdk/cosmos/azure-cosmos-tests/src/test/resources/multi-master-testng.xml index 65f6677c8fde..1bc9acb85918 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/resources/multi-master-testng.xml +++ b/sdk/cosmos/azure-cosmos-tests/src/test/resources/multi-master-testng.xml @@ -22,6 +22,9 @@ --> + + + diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/resources/multi-region-testng.xml b/sdk/cosmos/azure-cosmos-tests/src/test/resources/multi-region-testng.xml index 26b6eca2bcaa..f9dfd7eba045 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/resources/multi-region-testng.xml +++ b/sdk/cosmos/azure-cosmos-tests/src/test/resources/multi-region-testng.xml @@ -22,6 +22,9 @@ --> + + + diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/resources/query-testng.xml b/sdk/cosmos/azure-cosmos-tests/src/test/resources/query-testng.xml index d075671adf4d..4422c770175d 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/resources/query-testng.xml +++ b/sdk/cosmos/azure-cosmos-tests/src/test/resources/query-testng.xml @@ -22,6 +22,9 @@ --> + + + diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/resources/split-testng.xml b/sdk/cosmos/azure-cosmos-tests/src/test/resources/split-testng.xml index 9e6a54c38d90..65c6dec92ee6 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/resources/split-testng.xml +++ b/sdk/cosmos/azure-cosmos-tests/src/test/resources/split-testng.xml @@ -22,6 +22,9 @@ --> + + + diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/resources/thinclient-testng.xml b/sdk/cosmos/azure-cosmos-tests/src/test/resources/thinclient-testng.xml index ce233b954013..7e9ec9f26db8 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/resources/thinclient-testng.xml +++ b/sdk/cosmos/azure-cosmos-tests/src/test/resources/thinclient-testng.xml @@ -22,6 +22,9 @@ --> + + + diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/resources/unit-testng.xml b/sdk/cosmos/azure-cosmos-tests/src/test/resources/unit-testng.xml index 66fcce42608e..d92efb8abd4a 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/resources/unit-testng.xml +++ b/sdk/cosmos/azure-cosmos-tests/src/test/resources/unit-testng.xml @@ -22,6 +22,9 @@ --> + + + diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/AsyncDocumentClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/AsyncDocumentClient.java index f19ccb503027..9baeb17dbde8 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/AsyncDocumentClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/AsyncDocumentClient.java @@ -329,7 +329,14 @@ public AsyncDocumentClient build() { operationPolicies, isPerPartitionAutomaticFailoverEnabled); - client.init(state, null); + try { + client.init(state, null); + } catch (Throwable t) { + client.close(); + // TODO - should we map this to a CosmosException? + throw t; + } + return client; } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java index a66b7f05b425..50d85a95a05e 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java @@ -374,6 +374,9 @@ public class Configs { public static final String DEFAULT_OTEL_SPAN_ATTRIBUTE_NAMING_SCHEME = "All"; + private static final boolean DEFAULT_CLIENT_LEAK_DETECTION_ENABLED = false; + private static final String CLIENT_LEAK_DETECTION_ENABLED = "COSMOS.CLIENT_LEAK_DETECTION_ENABLED"; + public static int getCPUCnt() { return CPU_CNT; } @@ -492,6 +495,15 @@ public static boolean isThinClientEnabled() { return DEFAULT_THINCLIENT_ENABLED; } + public static boolean isClientLeakDetectionEnabled() { + String valueFromSystemProperty = System.getProperty(CLIENT_LEAK_DETECTION_ENABLED); + if (valueFromSystemProperty != null && !valueFromSystemProperty.isEmpty()) { + return Boolean.parseBoolean(valueFromSystemProperty); + } + + return DEFAULT_CLIENT_LEAK_DETECTION_ENABLED; + } + public int getUnavailableLocationsExpirationTimeInSeconds() { return getJVMConfigAsInt(UNAVAILABLE_LOCATIONS_EXPIRATION_TIME_IN_SECONDS, DEFAULT_UNAVAILABLE_LOCATIONS_EXPIRATION_TIME_IN_SECONDS); } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java index 28315f271bb4..9f6428ca5238 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java @@ -201,6 +201,14 @@ public class RxDocumentClientImpl implements AsyncDocumentClient, IAuthorization private static final String DUMMY_SQL_QUERY = "this is dummy and only used in creating " + "ParallelDocumentQueryExecutioncontext, but not used"; + private static final Object staticLock = new Object(); + + // A map containing the clientId with the callstack form where the Client was initialized. + // this can help to identify where clients leak. + // The leak detection via System property "COSMOS.CLIENT_LEAK_DETECTION_ENABLED" is disabled by + // default - CI pipeline tests will enable it. + private final static Map activeClients = new HashMap<>(); + private final static ObjectMapper mapper = Utils.getSimpleObjectMapper(); private final CosmosItemSerializer defaultCustomSerializer; private final static Logger logger = LoggerFactory.getLogger(RxDocumentClientImpl.class); @@ -1369,6 +1377,33 @@ private Flux> createQueryInternal( }, Queues.SMALL_BUFFER_SIZE, 1); } + private void addToActiveClients() { + if (Configs.isClientLeakDetectionEnabled()) { + synchronized (staticLock) { + activeClients.put(this.clientId, StackTraceUtil.currentCallStack()); + } + } + } + + private void removeFromActiveClients() { + if (Configs.isClientLeakDetectionEnabled()) { + synchronized (staticLock) { + activeClients.remove(this.clientId); + } + } + } + + /** + * Returns a snapshot of the active clients. The key is teh clientId, the value the callstack shows from + * where the client was created. + * @return a snapshot of the active clients. + */ + public static Map getActiveClientsSnapshot() { + synchronized (staticLock) { + return new HashMap<>(activeClients); + } + } + private static void applyExceptionToMergedDiagnosticsForQuery( CosmosQueryRequestOptions requestOptions, CosmosException exception, diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/StackTraceUtil.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/StackTraceUtil.java new file mode 100644 index 000000000000..bd262c067abc --- /dev/null +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/StackTraceUtil.java @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +package com.azure.cosmos.implementation; + +import java.util.Arrays; +import java.util.stream.Collectors; + +public final class StackTraceUtil { + private StackTraceUtil() {} + + /** Returns the current thread's call stack as a string (one frame per line). */ + public static String currentCallStack() { + return currentCallStack(0, Integer.MAX_VALUE, true); + } + + /** + * Returns the current thread's call stack as a string. + * + * @param skipFrames how many top frames to skip (in addition to this method & getStackTrace) + * @param maxFrames max number of frames to include + * @param trimInternal whether to drop the first 2 internal frames of this utility + */ + public static String currentCallStack(int skipFrames, int maxFrames, boolean trimInternal) { + StackTraceElement[] raw = Thread.currentThread().getStackTrace(); + int drop = (trimInternal ? 2 : 0) + Math.max(0, skipFrames); // drop getStackTrace + this method + return Arrays.stream(raw) + .skip(drop) + .limit(maxFrames) + .map(StackTraceUtil::formatFrame) + .collect(Collectors.joining(System.lineSeparator())); + } + + /** Formats a Throwable's stack trace into a single string (without the exception message). */ + public static String toString(Throwable t) { + return Arrays.stream(t.getStackTrace()) + .map(StackTraceUtil::formatFrame) + .collect(Collectors.joining(System.lineSeparator())); + } + + private static String formatFrame(StackTraceElement e) { + // e.g. "com.example.MyClass.myMethod(MyClass.java:42)" + String file = e.getFileName() == null ? "Unknown Source" : e.getFileName(); + String loc = e.getLineNumber() >= 0 ? (file + ":" + e.getLineNumber()) : file; + return e.getClassName() + "." + e.getMethodName() + "(" + loc + ")"; + } +} From 0e2032b3cc20fa1742a658b9b0258094ae326b49 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Fri, 7 Nov 2025 20:53:23 +0000 Subject: [PATCH 02/65] Update Configs.java --- .../src/main/java/com/azure/cosmos/implementation/Configs.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java index 50d85a95a05e..5de382166cf6 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java @@ -374,7 +374,7 @@ public class Configs { public static final String DEFAULT_OTEL_SPAN_ATTRIBUTE_NAMING_SCHEME = "All"; - private static final boolean DEFAULT_CLIENT_LEAK_DETECTION_ENABLED = false; + private static final boolean DEFAULT_CLIENT_LEAK_DETECTION_ENABLED = true; // TODO @fabianm - revert before merging - just cheaper to enable than making test config changes private static final String CLIENT_LEAK_DETECTION_ENABLED = "COSMOS.CLIENT_LEAK_DETECTION_ENABLED"; public static int getCPUCnt() { From 455b8bcc8aadf3c2d197ac8432455f1ceecbe76f Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Fri, 7 Nov 2025 21:25:54 +0000 Subject: [PATCH 03/65] Update CosmosNettyLeakDetectorFactory.java --- .../java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java index a889d700c555..95f032b97a77 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java @@ -19,7 +19,7 @@ public final class CosmosNettyLeakDetectorFactory extends ResourceLeakDetectorFa private static volatile boolean isLeakDetectionDisabled = false; private static volatile boolean isInitialized = false; - private CosmosNettyLeakDetectorFactory() { + public CosmosNettyLeakDetectorFactory() { } @Override From 8360f4971c26fa973a19ccc4c41df3476d7b8b37 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Fri, 7 Nov 2025 22:27:42 +0100 Subject: [PATCH 04/65] Update sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java index c4a602cedaff..d3a5fad98bfc 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java @@ -34,7 +34,7 @@ public void onExecutionFinish() { } // This method must be called as early as possible in the lifecycle of a process - // before any Netty ByteBNuf has been allocated + // before any Netty ByteBuf has been allocated public static void ingestIntoNetty() { if (isInitialized) { return; From f9f263c6150758be608a4225e1eae57efd941924 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Fri, 7 Nov 2025 21:31:54 +0000 Subject: [PATCH 05/65] Update CosmosNettyLeakDetectorFactory.java --- .../java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java index 95f032b97a77..df16902a9b9a 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java @@ -70,6 +70,8 @@ public static List resetIdentifiedLeaks() { public static AutoCloseable createDisableLeakDetectionScope() { synchronized (staticLock) { logger.info("Disabling Leak detection: {}", StackTraceUtil.currentCallStack()); + isLeakDetectionDisabled = true; + return new DisableLeakDetectionScope(); } } From 8e3a2d396b16610733a4022885a054ce93f7bfcb Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Fri, 7 Nov 2025 21:43:41 +0000 Subject: [PATCH 06/65] Fixes --- .../CosmosNettyLeakDetectorFactory.java | 47 +++++++++++------- .../cosmos/encryption/TestSuiteBase.java | 2 +- .../CosmosNettyLeakDetectorFactory.java | 49 ++++++++++++------- 3 files changed, 59 insertions(+), 39 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java index d3a5fad98bfc..ff1121f2a330 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java @@ -81,36 +81,45 @@ public ResourceLeakDetector newResourceLeakDetector( return new ResourceLeakDetector(resource, samplingInterval, maxActive) { @Override protected void reportTracedLeak(String resourceType, String records) { - synchronized (staticLock) { - if (!isLeakDetectionDisabled) { - String msg = "NETTY LEAK (traced) type=" - + resourceType - + "records=\n" - + records; - - identifiedLeaks.add(msg); - logger.error(msg); + if (!isLeakDetectionDisabled) { + synchronized (staticLock) { + if (!isLeakDetectionDisabled) { + String msg = "NETTY LEAK (traced) type=" + + resourceType + + "records=\n" + + records; + + identifiedLeaks.add(msg); + logger.error(msg); + } } } } @Override protected void reportUntracedLeak(String resourceType) { - synchronized (staticLock) { - String msg = "NETTY LEAK (untraced) type=" + resourceType; - - identifiedLeaks.add(msg); - logger.error(msg); + if (!isLeakDetectionDisabled) { + synchronized (staticLock) { + if (!isLeakDetectionDisabled) { + String msg = "NETTY LEAK (untraced) type=" + resourceType; + + identifiedLeaks.add(msg); + logger.error(msg); + } + } } } @Override protected void reportInstancesLeak(String resourceType) { - synchronized (staticLock) { - String msg = "NETTY LEAK (instances) type=" + resourceType; - - identifiedLeaks.add(msg); - logger.error(msg); + if (!isLeakDetectionDisabled) { + synchronized (staticLock) { + if (!isLeakDetectionDisabled) { + String msg = "NETTY LEAK (instances) type=" + resourceType; + identifiedLeaks.add(msg); + logger.error(msg); + } + } } } }; diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java index d121c33595c3..09cae0c5689f 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java @@ -275,7 +275,7 @@ public void afterClassSetupLeakDetection() { sb.append(leak).append("\n"); } - String msg = "\"NETTY LEAKS detected in test class: " + String msg = "NETTY LEAKS detected in test class: " + this.getClass().getCanonicalName() + sb; diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java index df16902a9b9a..fedef5451370 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java @@ -71,7 +71,7 @@ public static AutoCloseable createDisableLeakDetectionScope() { synchronized (staticLock) { logger.info("Disabling Leak detection: {}", StackTraceUtil.currentCallStack()); isLeakDetectionDisabled = true; - + return new DisableLeakDetectionScope(); } } @@ -83,36 +83,47 @@ public ResourceLeakDetector newResourceLeakDetector( return new ResourceLeakDetector(resource, samplingInterval, maxActive) { @Override protected void reportTracedLeak(String resourceType, String records) { - synchronized (staticLock) { - if (!isLeakDetectionDisabled) { - String msg = "NETTY LEAK (traced) type=" - + resourceType - + "records=\n" - + records; - - identifiedLeaks.add(msg); - logger.error(msg); + if (!isLeakDetectionDisabled) { + synchronized (staticLock) { + if (!isLeakDetectionDisabled) { + String msg = "NETTY LEAK (traced) type=" + + resourceType + + "records=\n" + + records; + + identifiedLeaks.add(msg); + logger.error(msg); + } } } } @Override protected void reportUntracedLeak(String resourceType) { - synchronized (staticLock) { - String msg = "NETTY LEAK (untraced) type=" + resourceType; - - identifiedLeaks.add(msg); - logger.error(msg); + if (!isLeakDetectionDisabled) { + synchronized (staticLock) { + if (!isLeakDetectionDisabled) { + String msg = "NETTY LEAK (untraced) type=" + resourceType; + + identifiedLeaks.add(msg); + logger.error(msg); + } + } } } @Override protected void reportInstancesLeak(String resourceType) { - synchronized (staticLock) { - String msg = "NETTY LEAK (instances) type=" + resourceType; + if (!isLeakDetectionDisabled) { + synchronized (staticLock) { + if (!isLeakDetectionDisabled) { + String msg = "NETTY LEAK (instances) type=" + resourceType; - identifiedLeaks.add(msg); - logger.error(msg); + identifiedLeaks.add(msg); + logger.error(msg); + + } + } } } }; From 13370c8e618dd64854869a4ca41098870ab3e7d8 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Fri, 7 Nov 2025 21:44:56 +0000 Subject: [PATCH 07/65] Fixes --- .../azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java | 2 +- .../java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java index ff1121f2a330..eba1e31491a0 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java @@ -86,7 +86,7 @@ protected void reportTracedLeak(String resourceType, String records) { if (!isLeakDetectionDisabled) { String msg = "NETTY LEAK (traced) type=" + resourceType - + "records=\n" + + " records=\n" + records; identifiedLeaks.add(msg); diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java index fedef5451370..5ce526e1841e 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java @@ -88,7 +88,7 @@ protected void reportTracedLeak(String resourceType, String records) { if (!isLeakDetectionDisabled) { String msg = "NETTY LEAK (traced) type=" + resourceType - + "records=\n" + + " records=\n" + records; identifiedLeaks.add(msg); From b59e147e09d5d9aebab8d7cce17d1c02867e18b2 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Fri, 7 Nov 2025 21:45:35 +0000 Subject: [PATCH 08/65] Update CosmosNettyLeakDetectorFactory.java --- .../java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java index 5ce526e1841e..b24feaedb2a0 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java @@ -34,7 +34,7 @@ public void onExecutionFinish() { } // This method must be called as early as possible in the lifecycle of a process - // before any Netty ByteBNuf has been allocated + // before any Netty ByteBuf has been allocated public static void ingestIntoNetty() { if (isInitialized) { return; From d3aa922416fe1b8bf0283d63b708198ecb81368f Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Fri, 7 Nov 2025 21:53:37 +0000 Subject: [PATCH 09/65] Update RxDocumentClientImpl.java --- .../com/azure/cosmos/implementation/RxDocumentClientImpl.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java index 9f6428ca5238..288a439752d0 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java @@ -514,6 +514,7 @@ private RxDocumentClientImpl(URI serviceEndpoint, this.sessionRetryOptions = sessionRetryOptions; this.defaultCustomSerializer = defaultCustomSerializer; + this.addToActiveClients(); logger.info( "Initializing DocumentClient [{}] with" + " serviceEndpoint [{}], connectionPolicy [{}], consistencyLevel [{}], readConsistencyStrategy [{}]", @@ -1394,7 +1395,7 @@ private void removeFromActiveClients() { } /** - * Returns a snapshot of the active clients. The key is teh clientId, the value the callstack shows from + * Returns a snapshot of the active clients. The key is the clientId, the value the callstack shows from * where the client was created. * @return a snapshot of the active clients. */ @@ -6475,6 +6476,7 @@ private RxStoreModel getStoreProxy(RxDocumentServiceRequest request) { @Override public void close() { + this.removeFromActiveClients(); logger.info("Attempting to close client {}", this.clientId); if (!closed.getAndSet(true)) { activeClientsCnt.decrementAndGet(); From 6c66864902a78f303648d6930efc098013503dae Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Fri, 7 Nov 2025 21:56:23 +0000 Subject: [PATCH 10/65] Fixes --- .../implementation/RxDocumentClientImpl.java | 2 +- .../cosmos/implementation/StackTraceUtil.java | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java index 288a439752d0..107f2e18911b 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java @@ -6476,9 +6476,9 @@ private RxStoreModel getStoreProxy(RxDocumentServiceRequest request) { @Override public void close() { - this.removeFromActiveClients(); logger.info("Attempting to close client {}", this.clientId); if (!closed.getAndSet(true)) { + this.removeFromActiveClients(); activeClientsCnt.decrementAndGet(); logger.info("Shutting down ..."); diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/StackTraceUtil.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/StackTraceUtil.java index bd262c067abc..af4387b33d04 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/StackTraceUtil.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/StackTraceUtil.java @@ -5,6 +5,21 @@ import java.util.Arrays; import java.util.stream.Collectors; +/** + * Utility class for capturing and formatting stack traces. + *

+ * This class provides methods to capture the current thread's call stack and format + * stack traces from Throwable objects. It's primarily used for diagnostic and debugging + * purposes, particularly in leak detection and client tracking scenarios. + *

+ *

+ * Note: Stack trace capture has performance implications and should be used judiciously, + * typically only when diagnostic features are enabled. + *

+ *

+ * This class is thread-safe as all methods operate on thread-local data or immutable parameters. + *

+ */ public final class StackTraceUtil { private StackTraceUtil() {} From 4dbfe6a53ee6090edd2ed4f9aaa83adc06b6d245 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Fri, 7 Nov 2025 22:11:45 +0000 Subject: [PATCH 11/65] Update CosmosNettyLeakDetectorFactory.java --- .../azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java index eba1e31491a0..4a786e1dbde2 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java @@ -70,6 +70,8 @@ public static List resetIdentifiedLeaks() { public static AutoCloseable createDisableLeakDetectionScope() { synchronized (staticLock) { logger.info("Disabling Leak detection: {}", StackTraceUtil.currentCallStack()); + isLeakDetectionDisabled = true; + return new DisableLeakDetectionScope(); } } From de7ecde2fe77356bac72c1768677fb3647661dcc Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Fri, 7 Nov 2025 22:25:33 +0000 Subject: [PATCH 12/65] Fixes --- .../java/com/azure/cosmos/encryption/TestSuiteBase.java | 6 +++++- .../src/test/resources/direct-testng.xml | 3 +++ .../src/test/resources/e2e-testng.xml | 3 +++ .../src/test/resources/emulator-testng.xml | 3 +++ .../src/test/resources/encryption-testng.xml | 3 +++ .../src/test/resources/examples-testng.xml | 3 +++ .../src/test/resources/fast-testng.xml | 3 +++ .../src/test/resources/long-testng.xml | 3 +++ .../src/test/resources/multi-master-testng.xml | 3 +++ .../src/test/resources/unit-testng.xml | 3 +++ .../java/com/azure/cosmos/implementation/TestSuiteBase.java | 2 +- .../src/test/java/com/azure/cosmos/rx/TestSuiteBase.java | 2 +- 12 files changed, 34 insertions(+), 3 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java index 09cae0c5689f..77afa574c115 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java @@ -237,7 +237,6 @@ public CosmosAsyncDatabase getDatabase(String id) { @BeforeClass(groups = {"fast", "long", "direct", "multi-master", "encryption"}, timeOut = SUITE_SETUP_TIMEOUT) public void beforeClassSetupLeakDetection() { - CosmosNettyLeakDetectorFactory.ingestIntoNetty(); this.activeClientsAtBegin = RxDocumentClientImpl.getActiveClientsSnapshot(); } @@ -1544,4 +1543,9 @@ protected static void validateResponse(EncryptionPojo originalItem, EncryptionPo assertThat(result.getSensitiveChildPojo2DArray()[0][0].getSensitiveStringArray()).isEqualTo(originalItem.getSensitiveChildPojo2DArray()[0][0].getSensitiveStringArray()); assertThat(result.getSensitiveChildPojo2DArray()[0][0].getSensitiveString3DArray()).isEqualTo(originalItem.getSensitiveChildPojo2DArray()[0][0].getSensitiveString3DArray()); } + + + static { + CosmosNettyLeakDetectorFactory.ingestIntoNetty(); + } } diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/resources/direct-testng.xml b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/direct-testng.xml index 293d3030e9e9..cf17a626a770 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/resources/direct-testng.xml +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/direct-testng.xml @@ -22,6 +22,9 @@ --> + + + diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/resources/e2e-testng.xml b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/e2e-testng.xml index 17b22dc377f5..ff6a7907d5f9 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/resources/e2e-testng.xml +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/e2e-testng.xml @@ -22,6 +22,9 @@ --> + + + diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/resources/emulator-testng.xml b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/emulator-testng.xml index 59d8973d566c..98c9ff76bfb3 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/resources/emulator-testng.xml +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/emulator-testng.xml @@ -22,6 +22,9 @@ --> + + + diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/resources/encryption-testng.xml b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/encryption-testng.xml index 1a0f0feaae94..749863140a9e 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/resources/encryption-testng.xml +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/encryption-testng.xml @@ -22,6 +22,9 @@ --> + + + diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/resources/examples-testng.xml b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/examples-testng.xml index cc10ecf164d9..5bf69639bb73 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/resources/examples-testng.xml +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/examples-testng.xml @@ -22,6 +22,9 @@ --> + + + diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/resources/fast-testng.xml b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/fast-testng.xml index 3e241e961493..9a6b63cbf109 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/resources/fast-testng.xml +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/fast-testng.xml @@ -22,6 +22,9 @@ --> + + + diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/resources/long-testng.xml b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/long-testng.xml index d19f387a130b..362904fb462e 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/resources/long-testng.xml +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/long-testng.xml @@ -22,6 +22,9 @@ --> + + + diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/resources/multi-master-testng.xml b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/multi-master-testng.xml index 65f6677c8fde..f83dc6cdd57e 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/resources/multi-master-testng.xml +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/multi-master-testng.xml @@ -22,6 +22,9 @@ --> + + + diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/resources/unit-testng.xml b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/unit-testng.xml index 66fcce42608e..b1abc6f0e645 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/resources/unit-testng.xml +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/unit-testng.xml @@ -22,6 +22,9 @@ --> + + + diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestSuiteBase.java index 19f3c1552316..9b0e39dbfe61 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestSuiteBase.java @@ -95,6 +95,7 @@ private static ImmutableList immutableListOrNull(List list) { } static { + CosmosNettyLeakDetectorFactory.ingestIntoNetty(); accountConsistency = parseConsistency(TestConfigurations.CONSISTENCY); desiredConsistencies = immutableListOrNull( ObjectUtils.defaultIfNull(parseDesiredConsistencies(TestConfigurations.DESIRED_CONSISTENCIES), @@ -154,7 +155,6 @@ public Mono> deleteDatabase(String id) { "split", "query", "cfp-split", "long-emulator"}, timeOut = SUITE_SETUP_TIMEOUT) public void beforeClassSetupLeakDetection() { - CosmosNettyLeakDetectorFactory.ingestIntoNetty(); this.activeClientsAtBegin = RxDocumentClientImpl.getActiveClientsSnapshot(); } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java index bbc4241d9d95..7ecc1ea89567 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java @@ -162,6 +162,7 @@ protected static CosmosAsyncContainer getSharedSinglePartitionCosmosContainer(Co } static { + CosmosNettyLeakDetectorFactory.ingestIntoNetty(); accountConsistency = parseConsistency(TestConfigurations.CONSISTENCY); desiredConsistencies = immutableListOrNull( ObjectUtils.defaultIfNull(parseDesiredConsistencies(TestConfigurations.DESIRED_CONSISTENCIES), @@ -218,7 +219,6 @@ public CosmosAsyncDatabase getDatabase(String id) { "emulator-vnext", "split", "query", "cfp-split", "circuit-breaker-misc-gateway", "circuit-breaker-misc-direct", "circuit-breaker-read-all-read-many", "fi-multi-master", "long-emulator", "fi-thinclient-multi-region", "fi-thinclient-multi-master"}, timeOut = SUITE_SETUP_TIMEOUT) public void beforeClassSetupLeakDetection() { - CosmosNettyLeakDetectorFactory.ingestIntoNetty(); this.activeClientsAtBegin = RxDocumentClientImpl.getActiveClientsSnapshot(); } From bb9493d8b9d16e30654494398f4fe183e75c0e39 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Fri, 7 Nov 2025 22:27:50 +0000 Subject: [PATCH 13/65] Fixes --- .../cosmos/encryption/CosmosNettyLeakDetectorFactory.java | 3 ++- .../java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java index 4a786e1dbde2..ac7fa0481d26 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java @@ -19,7 +19,8 @@ public final class CosmosNettyLeakDetectorFactory extends ResourceLeakDetectorFa private static volatile boolean isLeakDetectionDisabled = false; private static volatile boolean isInitialized = false; - private CosmosNettyLeakDetectorFactory() { + public CosmosNettyLeakDetectorFactory() { + ingestIntoNetty(); } @Override diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java index b24feaedb2a0..3b2a4b383086 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java @@ -20,6 +20,7 @@ public final class CosmosNettyLeakDetectorFactory extends ResourceLeakDetectorFa private static volatile boolean isInitialized = false; public CosmosNettyLeakDetectorFactory() { + ingestIntoNetty(); } @Override From 2e9c533e53a348f0f27cb657d16c76d06ffcd481 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Fri, 7 Nov 2025 22:46:10 +0000 Subject: [PATCH 14/65] Iterating on tests --- .../CosmosNettyLeakDetectorFactory.java | 13 +++++- .../cosmos/CosmosDiagnosticsE2ETest.java | 43 +++++++++++++------ .../CosmosNettyLeakDetectorFactory.java | 13 +++++- 3 files changed, 53 insertions(+), 16 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java index ac7fa0481d26..6587dceb337f 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java @@ -20,7 +20,6 @@ public final class CosmosNettyLeakDetectorFactory extends ResourceLeakDetectorFa private static volatile boolean isInitialized = false; public CosmosNettyLeakDetectorFactory() { - ingestIntoNetty(); } @Override @@ -32,6 +31,11 @@ public void onExecutionStart() { public void onExecutionFinish() { // Run GC to force finalizers to run - only in finalizers Netty would actually detect any leaks. System.gc(); + try { + Thread.sleep(1_000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } } // This method must be called as early as possible in the lifecycle of a process @@ -53,6 +57,7 @@ public static void ingestIntoNetty() { // install custom reporter ResourceLeakDetectorFactory.setResourceLeakDetectorFactory(new CosmosNettyLeakDetectorFactory()); + logger.info("NETTY LEAK detection initialized"); isInitialized = true; } } @@ -60,6 +65,12 @@ public static void ingestIntoNetty() { public static List resetIdentifiedLeaks() { // Run GC to force finalizers to run - only in finalizers Netty would actually detect any leaks. System.gc(); + try { + Thread.sleep(1_000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + synchronized (staticLock) { List leaksSnapshot = new ArrayList<>(identifiedLeaks); diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosDiagnosticsE2ETest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosDiagnosticsE2ETest.java index d72bd1962fcf..786cfb12d249 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosDiagnosticsE2ETest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosDiagnosticsE2ETest.java @@ -26,6 +26,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import io.micrometer.core.instrument.MeterRegistry; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.ByteBufAllocator; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; @@ -154,12 +156,16 @@ public void onlyLoggerWithCustomConfig() { .diagnosticsHandler(CosmosDiagnosticsHandler.DEFAULT_LOGGING_HANDLER) ); CosmosContainer container = this.getContainer(builder); - CosmosItemResponse response = executeTestCase(container); - assertThat(response.getDiagnostics()).isNotNull(); - assertThat(response.getDiagnostics().getDiagnosticsContext()).isNotNull(); - // no assertions here - invocations for diagnostics handler are validated above - // log4j event logging isn't validated in general in unit tests because it is too brittle to do so - // with custom appender + try { + CosmosItemResponse response = executeTestCase(container); + assertThat(response.getDiagnostics()).isNotNull(); + assertThat(response.getDiagnostics().getDiagnosticsContext()).isNotNull(); + // no assertions here - invocations for diagnostics handler are validated above + // log4j event logging isn't validated in general in unit tests because it is too brittle to do so + // with custom appender + } finally { + safeCloseCosmosClient(); + } } @Test(groups = { "fast", "emulator" }, timeOut = TIMEOUT) @@ -436,12 +442,17 @@ public void validateFeedOperationStateNullCheck() { CosmosAsyncClient client = this .getClientBuilder() .buildAsyncClient(); - TestUtils.createDummyQueryFeedOperationStateWithoutPagedFluxOptions( - ResourceType.Document, - OperationType.Query, - new CosmosQueryRequestOptions(), - client - ); + + try { + TestUtils.createDummyQueryFeedOperationStateWithoutPagedFluxOptions( + ResourceType.Document, + OperationType.Query, + new CosmosQueryRequestOptions(), + client + ); + } finally { + client.close(); + } } private CosmosItemResponse executeTestCase(CosmosContainer container) { @@ -472,12 +483,16 @@ private ObjectNode getDocumentDefinition(String documentId) { } } - private CosmosContainer getContainer(CosmosClientBuilder builder) { - + private void safeCloseCosmosClient() { CosmosClient oldClient = this.client; if (oldClient != null) { oldClient.close(); } + } + + private CosmosContainer getContainer(CosmosClientBuilder builder) { + + this.safeCloseCosmosClient(); assertThat(builder).isNotNull(); this.client = builder.buildClient(); diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java index 3b2a4b383086..de37d6e1cb11 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java @@ -20,7 +20,6 @@ public final class CosmosNettyLeakDetectorFactory extends ResourceLeakDetectorFa private static volatile boolean isInitialized = false; public CosmosNettyLeakDetectorFactory() { - ingestIntoNetty(); } @Override @@ -32,6 +31,11 @@ public void onExecutionStart() { public void onExecutionFinish() { // Run GC to force finalizers to run - only in finalizers Netty would actually detect any leaks. System.gc(); + try { + Thread.sleep(1_000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } } // This method must be called as early as possible in the lifecycle of a process @@ -53,6 +57,7 @@ public static void ingestIntoNetty() { // install custom reporter ResourceLeakDetectorFactory.setResourceLeakDetectorFactory(new CosmosNettyLeakDetectorFactory()); + logger.info("NETTY LEAK detection initialized"); isInitialized = true; } } @@ -60,6 +65,12 @@ public static void ingestIntoNetty() { public static List resetIdentifiedLeaks() { // Run GC to force finalizers to run - only in finalizers Netty would actually detect any leaks. System.gc(); + try { + Thread.sleep(1_000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + synchronized (staticLock) { List leaksSnapshot = new ArrayList<>(identifiedLeaks); From 6ac7997482b611577484765f71e91b56c5e57ac7 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Fri, 7 Nov 2025 23:30:25 +0000 Subject: [PATCH 15/65] Fixing build warning --- .../java/com/azure/cosmos/encryption/TestSuiteBase.java | 6 +++--- sdk/cosmos/azure-cosmos-kafka-connect/pom.xml | 5 +++++ .../connect/KafkaCosmosIntegrationTestSuiteBase.java | 4 ++-- .../cosmos/kafka/connect/KafkaCosmosTestSuiteBase.java | 8 ++++---- .../com/azure/cosmos/implementation/TestSuiteBase.java | 6 +++--- .../src/test/java/com/azure/cosmos/rx/TestSuiteBase.java | 2 +- 6 files changed, 18 insertions(+), 13 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java index 77afa574c115..5ac838c0b9a8 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java @@ -284,7 +284,7 @@ public void afterClassSetupLeakDetection() { } @BeforeSuite(groups = {"fast", "long", "direct", "multi-master", "encryption"}, timeOut = SUITE_SETUP_TIMEOUT) - public static void beforeSuite() { + public void beforeSuite() { logger.info("beforeSuite Started"); @@ -318,14 +318,14 @@ public static void beforeSuite() { } @BeforeSuite(groups = {"unit"}) - public static void parallelizeUnitTests(ITestContext context) { + public void parallelizeUnitTests(ITestContext context) { // TODO: Parallelization was disabled due to flaky tests. Re-enable after fixing the flaky tests. // context.getSuite().getXmlSuite().setParallel(XmlSuite.ParallelMode.CLASSES); // context.getSuite().getXmlSuite().setThreadCount(Runtime.getRuntime().availableProcessors()); } @AfterSuite(groups = {"fast", "long", "direct", "multi-master", "encryption"}, timeOut = SUITE_SHUTDOWN_TIMEOUT) - public static void afterSuite() { + public void afterSuite() { logger.info("afterSuite Started"); diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml b/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml index b61863fd3191..1693af8dc8fb 100644 --- a/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml +++ b/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml @@ -32,6 +32,11 @@ Licensed under the MIT License. Confluent https://packages.confluent.io/maven/ + + maven-repo1 + Maven Repo1 + https://repo1.maven.org/maven2/ + diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosIntegrationTestSuiteBase.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosIntegrationTestSuiteBase.java index c075f7e397d3..68f7b87b7a93 100644 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosIntegrationTestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosIntegrationTestSuiteBase.java @@ -39,7 +39,7 @@ public class KafkaCosmosIntegrationTestSuiteBase extends KafkaCosmosTestSuiteBas protected static KafkaCosmosConnectContainer kafkaCosmosConnectContainer; @BeforeSuite(groups = { "kafka-integration" }, timeOut = 10 * SUITE_SETUP_TIMEOUT) - public static void beforeIntegrationSuite() throws IOException, InterruptedException { + public void beforeIntegrationSuite() throws IOException, InterruptedException { logger.info("beforeIntegrationSuite Started"); // initialize the kafka, kafka-connect containers @@ -47,7 +47,7 @@ public static void beforeIntegrationSuite() throws IOException, InterruptedExcep } @AfterSuite(groups = { "kafka-integration" }, timeOut = 10 * SUITE_SETUP_TIMEOUT) - public static void afterIntegrationSuite() { + public void afterIntegrationSuite() { logger.info("afterIntegrationSuite Started"); // The TestContainers library will automatically clean up resources by using Ryuk sidecar container diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosTestSuiteBase.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosTestSuiteBase.java index 68f0a16f361c..56d51facc78c 100644 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosTestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosTestSuiteBase.java @@ -90,7 +90,7 @@ protected static CosmosContainerProperties getSinglePartitionContainer(CosmosAsy } @BeforeSuite(groups = { "kafka", "kafka-integration" }, timeOut = SUITE_SETUP_TIMEOUT) - public static void beforeSuite() { + public void beforeSuite() { logger.info("beforeSuite Started"); try (CosmosAsyncClient houseKeepingClient = createGatewayHouseKeepingDocumentClient(true).buildAsyncClient()) { @@ -122,7 +122,7 @@ public static void beforeSuite() { } @BeforeSuite(groups = { "kafka-emulator" }, timeOut = SUITE_SETUP_TIMEOUT) - public static void beforeSuite_emulator() { + public void beforeSuite_emulator() { logger.info("beforeSuite Started"); try (CosmosAsyncClient houseKeepingClient = createGatewayHouseKeepingDocumentClient(true).buildAsyncClient()) { @@ -154,7 +154,7 @@ public static void beforeSuite_emulator() { } @BeforeSuite(groups = { "unit" }, timeOut = SUITE_SETUP_TIMEOUT) - public static void beforeSuite_unit() { + public void beforeSuite_unit() { logger.info("beforeSuite for unit tests started"); databaseName = @@ -166,7 +166,7 @@ public static void beforeSuite_unit() { } @AfterSuite(groups = { "kafka", "kafka-integration", "kafka-emulator" }, timeOut = SUITE_SHUTDOWN_TIMEOUT) - public static void afterSuite() { + public void afterSuite() { logger.info("afterSuite Started"); diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestSuiteBase.java index 9b0e39dbfe61..588658444b08 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestSuiteBase.java @@ -204,7 +204,7 @@ public void afterClassSetupLeakDetection() { @BeforeSuite(groups = {"fast", "long", "direct", "multi-region", "multi-master", "flaky-multi-master", "emulator", "split", "query", "cfp-split", "long-emulator"}, timeOut = SUITE_SETUP_TIMEOUT) - public static void beforeSuite() { + public void beforeSuite() { logger.info("beforeSuite Started"); AsyncDocumentClient houseKeepingClient = createGatewayHouseKeepingDocumentClient().build(); try { @@ -221,7 +221,7 @@ public static void beforeSuite() { } @BeforeSuite(groups = {"unit"}) - public static void parallelizeUnitTests(ITestContext context) { + public void parallelizeUnitTests(ITestContext context) { // TODO: Parallelization was disabled due to flaky tests. Re-enable after fixing the flaky tests. // context.getSuite().getXmlSuite().setParallel(XmlSuite.ParallelMode.CLASSES); // context.getSuite().getXmlSuite().setThreadCount(Runtime.getRuntime().availableProcessors()); @@ -229,7 +229,7 @@ public static void parallelizeUnitTests(ITestContext context) { @AfterSuite(groups = {"fast", "long", "direct", "multi-region", "multi-master", "flaky-multi-master", "emulator", "split", "query", "cfp-split", "long-emulator"}, timeOut = SUITE_SHUTDOWN_TIMEOUT) - public static void afterSuite() { + public void afterSuite() { logger.info("afterSuite Started"); AsyncDocumentClient houseKeepingClient = createGatewayHouseKeepingDocumentClient().build(); try { diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java index 7ecc1ea89567..c8217f793afc 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java @@ -286,7 +286,7 @@ public void beforeSuite() { } @BeforeSuite(groups = {"unit"}) - public static void parallelizeUnitTests(ITestContext context) { + public void parallelizeUnitTests(ITestContext context) { // TODO: Parallelization was disabled due to flaky tests. Re-enable after fixing the flaky tests. // context.getSuite().getXmlSuite().setParallel(XmlSuite.ParallelMode.CLASSES); // context.getSuite().getXmlSuite().setThreadCount(Runtime.getRuntime().availableProcessors()); From 190b9e383166315608cc8db2a3c96ae2d554ba24 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Sat, 8 Nov 2025 01:15:34 +0000 Subject: [PATCH 16/65] Fixing memory leak --- .../CosmosNettyLeakDetectorFactory.java | 1 + .../CosmosNettyLeakDetectorFactory.java | 1 + .../implementation/RxGatewayStoreModel.java | 66 +++++++++++-------- 3 files changed, 42 insertions(+), 26 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java index 6587dceb337f..c2b5f179131f 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java @@ -54,6 +54,7 @@ public static void ingestIntoNetty() { ResourceLeakDetector.setLevel(ResourceLeakDetector.Level.PARANOID); // sample every allocation System.setProperty("io.netty.leakDetection.samplingInterval", "1"); + System.setProperty("io.netty.leakDetection.targetRecords", "256"); // install custom reporter ResourceLeakDetectorFactory.setResourceLeakDetectorFactory(new CosmosNettyLeakDetectorFactory()); diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java index de37d6e1cb11..c94a542edfff 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java @@ -54,6 +54,7 @@ public static void ingestIntoNetty() { ResourceLeakDetector.setLevel(ResourceLeakDetector.Level.PARANOID); // sample every allocation System.setProperty("io.netty.leakDetection.samplingInterval", "1"); + System.setProperty("io.netty.leakDetection.targetRecords", "256"); // install custom reporter ResourceLeakDetectorFactory.setResourceLeakDetectorFactory(new CosmosNettyLeakDetectorFactory()); diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java index ea81761b0411..7143b0ad8427 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java @@ -417,7 +417,12 @@ private Mono toDocumentServiceResponse(Mono leakDetectionDebuggingEnabled ? bodyByteBuf.retain().touch(this) : bodyByteBuf.retain()) - .publishOn(CosmosSchedulers.TRANSPORT_RESPONSE_BOUNDED_ELASTIC); + .publishOn(CosmosSchedulers.TRANSPORT_RESPONSE_BOUNDED_ELASTIC) + .doOnDiscard(ByteBuf.class, buf -> { + if (buf.refCnt() > 0) { + io.netty.util.ReferenceCountUtil.safeRelease(buf); + } + }); return contentObservable .map(content -> { @@ -425,38 +430,47 @@ private Mono toDocumentServiceResponse(Mono 0) { + // Unwrap failed before StoreResponse took ownership -> release our retain + io.netty.util.ReferenceCountUtil.safeRelease(content); + } + + throw t; + } }) .single(); From 8c704640af46a0b836fd3db3efa4eb6a4a91e4e2 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Mon, 10 Nov 2025 08:55:45 +0000 Subject: [PATCH 17/65] Reverting production changes --- .../implementation/AsyncDocumentClient.java | 9 +-- .../implementation/RxGatewayStoreModel.java | 66 ++++++++----------- 2 files changed, 27 insertions(+), 48 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/AsyncDocumentClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/AsyncDocumentClient.java index 9baeb17dbde8..f19ccb503027 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/AsyncDocumentClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/AsyncDocumentClient.java @@ -329,14 +329,7 @@ public AsyncDocumentClient build() { operationPolicies, isPerPartitionAutomaticFailoverEnabled); - try { - client.init(state, null); - } catch (Throwable t) { - client.close(); - // TODO - should we map this to a CosmosException? - throw t; - } - + client.init(state, null); return client; } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java index 7143b0ad8427..ea81761b0411 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java @@ -417,12 +417,7 @@ private Mono toDocumentServiceResponse(Mono leakDetectionDebuggingEnabled ? bodyByteBuf.retain().touch(this) : bodyByteBuf.retain()) - .publishOn(CosmosSchedulers.TRANSPORT_RESPONSE_BOUNDED_ELASTIC) - .doOnDiscard(ByteBuf.class, buf -> { - if (buf.refCnt() > 0) { - io.netty.util.ReferenceCountUtil.safeRelease(buf); - } - }); + .publishOn(CosmosSchedulers.TRANSPORT_RESPONSE_BOUNDED_ELASTIC); return contentObservable .map(content -> { @@ -430,47 +425,38 @@ private Mono toDocumentServiceResponse(Mono 0) { - // Unwrap failed before StoreResponse took ownership -> release our retain - io.netty.util.ReferenceCountUtil.safeRelease(content); + rsp.setFaultInjectionRuleEvaluationResults( + request + .faultInjectionRequestContext + .getFaultInjectionRuleEvaluationResults(reactorNettyRequestRecord.getTransportRequestId())); } + } - throw t; + if (request.requestContext.cosmosDiagnostics != null) { + BridgeInternal.recordGatewayResponse(request.requestContext.cosmosDiagnostics, request, rsp, globalEndpointManager); } + + return rsp; }) .single(); From bca461faea01f9beec2a20a08b4820cd678d2bca Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Mon, 10 Nov 2025 12:11:17 +0000 Subject: [PATCH 18/65] Iterating on test tools --- .../CosmosEncryptionAsyncClientTest.java | 89 ++++++++++++++++++ .../cosmos/encryption/TestSuiteBase.java | 61 ------------ .../azure/cosmos/CosmosAsyncClientTest.java | 92 +++++++++++++++++++ .../com/azure/cosmos/DocumentClientTest.java | 92 ++++++++++++++++++- .../RegionScopedSessionContainerTest.java | 4 +- .../cosmos/implementation/TestSuiteBase.java | 62 ------------- .../com/azure/cosmos/rx/TestSuiteBase.java | 64 ------------- 7 files changed, 275 insertions(+), 189 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncClientTest.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncClientTest.java index e4856585a9d2..0c39b750776d 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncClientTest.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncClientTest.java @@ -9,19 +9,36 @@ import com.azure.cosmos.implementation.Configs; import com.azure.cosmos.implementation.ConnectionPolicy; import com.azure.cosmos.implementation.ImplementationBridgeHelpers; +import com.azure.cosmos.implementation.RxDocumentClientImpl; import com.azure.cosmos.implementation.guava27.Strings; +import io.netty.buffer.PooledByteBufAllocator; +import io.netty.util.internal.PlatformDependent; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.testng.ITest; +import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; +import java.lang.management.BufferPoolMXBean; +import java.lang.management.ManagementFactory; import java.lang.reflect.Method; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; public abstract class CosmosEncryptionAsyncClientTest implements ITest { + protected static Logger logger = LoggerFactory.getLogger(CosmosEncryptionAsyncClientTest.class.getSimpleName()); + protected static final int SUITE_SETUP_TIMEOUT = 120000; private static final ImplementationBridgeHelpers.CosmosClientBuilderHelper.CosmosClientBuilderAccessor cosmosClientBuilderAccessor = ImplementationBridgeHelpers.CosmosClientBuilderHelper.getCosmosClientBuilderAccessor(); + private final static AtomicInteger instancesUsed = new AtomicInteger(0); private final CosmosClientBuilder clientBuilder; private String testName; + private volatile Map activeClientsAtBegin = new HashMap<>(); public CosmosEncryptionAsyncClientTest() { this(new CosmosClientBuilder()); @@ -35,6 +52,78 @@ public final CosmosClientBuilder getClientBuilder() { return this.clientBuilder; } + @BeforeClass(groups = {"fast", "long", "direct", "multi-master", "encryption"}, timeOut = SUITE_SETUP_TIMEOUT) + + public void beforeClassSetupLeakDetection() { + if (instancesUsed.getAndIncrement() == 0) { + this.activeClientsAtBegin = RxDocumentClientImpl.getActiveClientsSnapshot(); + this.logMemoryUsage("BEFORE"); + } + } + + private void logMemoryUsage(String name) { + long pooledDirectBytes = PooledByteBufAllocator.DEFAULT.metric() + .directArenas().stream() + .mapToLong(io.netty.buffer.PoolArenaMetric::numActiveBytes) + .sum(); + + long used = PlatformDependent.usedDirectMemory(); + long max = PlatformDependent.maxDirectMemory(); + logger.info("MEMORY USAGE: {}:{}", this.getClass().getCanonicalName(), name); + logger.info("Netty Direct Memory: {}/{}/{} bytes", used, pooledDirectBytes, max); + for (BufferPoolMXBean pool : ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class)) { + logger.info("Pool {}: used={} bytes, capacity={} bytes, count={}", + pool.getName(), pool.getMemoryUsed(), pool.getTotalCapacity(), pool.getCount()); + } + } + + @AfterClass(groups = {"fast", "long", "direct", "multi-master", "encryption"}, timeOut = SUITE_SETUP_TIMEOUT) + public void afterClassSetupLeakDetection() { + if (instancesUsed.decrementAndGet() == 0) { + Map leakedClientSnapshotNow = RxDocumentClientImpl.getActiveClientsSnapshot(); + StringBuilder sb = new StringBuilder(); + Map leakedClientSnapshotAtBegin = activeClientsAtBegin; + + for (Integer clientId : leakedClientSnapshotNow.keySet()) { + if (!leakedClientSnapshotAtBegin.containsKey(clientId)) { + // this client was leaked in this class + sb + .append("CosmosClient [") + .append(clientId) + .append("] leaked. Callstack of initialization:\n") + .append(leakedClientSnapshotNow.get(clientId)) + .append("\n\n"); + } + } + + if (sb.length() > 0) { + String msg = "\"COSMOS CLIENT LEAKS detected in test class: " + + this.getClass().getCanonicalName() + + "\n\n" + + sb; + + logger.error(msg); + // fail(msg); + } + + List nettyLeaks = CosmosNettyLeakDetectorFactory.resetIdentifiedLeaks(); + if (nettyLeaks.size() > 0) { + sb.append("\n"); + for (String leak : nettyLeaks) { + sb.append(leak).append("\n"); + } + + String msg = "NETTY LEAKS detected in test class: " + + this.getClass().getCanonicalName() + + sb; + + logger.error(msg); + // fail(msg); + } + this.logMemoryUsage("AFTER"); + } + } + @Override public final String getTestName() { return this.testName; diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java index 5ac838c0b9a8..3457351177d4 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java @@ -26,7 +26,6 @@ import com.azure.cosmos.implementation.ConnectionPolicy; import com.azure.cosmos.implementation.InternalObjectNode; import com.azure.cosmos.implementation.PathParser; -import com.azure.cosmos.implementation.RxDocumentClientImpl; import com.azure.cosmos.implementation.TestConfigurations; import com.azure.cosmos.implementation.Utils; import com.azure.cosmos.implementation.directconnectivity.Protocol; @@ -60,12 +59,8 @@ import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.mockito.stubbing.Answer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.testng.ITestContext; -import org.testng.annotations.AfterClass; import org.testng.annotations.AfterSuite; -import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeSuite; import org.testng.annotations.DataProvider; import org.testng.annotations.Listeners; @@ -76,9 +71,7 @@ import java.time.Duration; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -86,7 +79,6 @@ import static com.azure.cosmos.BridgeInternal.extractConfigs; import static com.azure.cosmos.BridgeInternal.injectConfigs; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Fail.fail; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.spy; @@ -96,13 +88,11 @@ public class TestSuiteBase extends CosmosEncryptionAsyncClientTest { private static final int DEFAULT_BULK_INSERT_CONCURRENCY_LEVEL = 500; private static final ObjectMapper objectMapper = new ObjectMapper(); - protected static Logger logger = LoggerFactory.getLogger(TestSuiteBase.class.getSimpleName()); protected static final int TIMEOUT = 40000; protected static final int FEED_TIMEOUT = 40000; protected static final int SETUP_TIMEOUT = 60000; protected static final int SHUTDOWN_TIMEOUT = 24000; - protected static final int SUITE_SETUP_TIMEOUT = 120000; protected static final int SUITE_SHUTDOWN_TIMEOUT = 60000; protected static final int WAIT_REPLICA_CATCH_UP_IN_MILLIS = 4000; @@ -116,8 +106,6 @@ public class TestSuiteBase extends CosmosEncryptionAsyncClientTest { protected int subscriberValidationTimeout = TIMEOUT; - private volatile Map activeClientsAtBegin = new HashMap<>(); - private static CosmosAsyncDatabase SHARED_DATABASE; private static CosmosAsyncContainer SHARED_MULTI_PARTITION_COLLECTION_WITH_ID_AS_PARTITION_KEY; private static CosmosAsyncContainer SHARED_MULTI_PARTITION_COLLECTION; @@ -234,55 +222,6 @@ public CosmosAsyncDatabase getDatabase(String id) { } } - @BeforeClass(groups = {"fast", "long", "direct", "multi-master", "encryption"}, timeOut = SUITE_SETUP_TIMEOUT) - - public void beforeClassSetupLeakDetection() { - this.activeClientsAtBegin = RxDocumentClientImpl.getActiveClientsSnapshot(); - } - - @AfterClass(groups = {"fast", "long", "direct", "multi-master", "encryption"}, timeOut = SUITE_SETUP_TIMEOUT) - public void afterClassSetupLeakDetection() { - - Map leakedClientSnapshotNow = RxDocumentClientImpl.getActiveClientsSnapshot(); - StringBuilder sb = new StringBuilder(); - Map leakedClientSnapshotAtBegin = activeClientsAtBegin; - - for (Integer clientId : leakedClientSnapshotNow.keySet()) { - if (!leakedClientSnapshotAtBegin.containsKey(clientId)) { - // this client was leaked in this class - sb - .append("CosmosClient [") - .append(clientId) - .append("] leaked. Callstack of initialization:\n") - .append(leakedClientSnapshotNow.get(clientId)); - } - } - - if (sb.length() > 0) { - String msg = "\"COSMOS CLIENT LEAKS detected in test class: " - + this.getClass().getCanonicalName() - + sb; - - logger.error(msg); - // fail(msg); - } - - List nettyLeaks = CosmosNettyLeakDetectorFactory.resetIdentifiedLeaks(); - if (nettyLeaks.size() > 0) { - sb.append("\n"); - for (String leak : nettyLeaks) { - sb.append(leak).append("\n"); - } - - String msg = "NETTY LEAKS detected in test class: " - + this.getClass().getCanonicalName() - + sb; - - logger.error(msg); - // fail(msg); - } - } - @BeforeSuite(groups = {"fast", "long", "direct", "multi-master", "encryption"}, timeOut = SUITE_SETUP_TIMEOUT) public void beforeSuite() { diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosAsyncClientTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosAsyncClientTest.java index a1bfb4abf41b..554b52f3804c 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosAsyncClientTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosAsyncClientTest.java @@ -4,16 +4,28 @@ package com.azure.cosmos; import com.azure.cosmos.implementation.ConnectionPolicy; +import com.azure.cosmos.implementation.RxDocumentClientImpl; import com.azure.cosmos.implementation.guava27.Strings; import com.azure.cosmos.models.CosmosItemRequestOptions; import com.azure.cosmos.models.CosmosItemResponse; import com.azure.cosmos.models.PartitionKey; +import io.netty.buffer.PooledByteBufAllocator; +import io.netty.util.internal.PlatformDependent; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.testng.ITest; +import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; +import java.lang.management.BufferPoolMXBean; +import java.lang.management.ManagementFactory; import java.lang.reflect.Method; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; import static org.assertj.core.api.Assertions.assertThat; @@ -21,8 +33,13 @@ public abstract class CosmosAsyncClientTest implements ITest { public static final String ROUTING_GATEWAY_EMULATOR_PORT = ":8081"; public static final String COMPUTE_GATEWAY_EMULATOR_PORT = ":9999"; + + protected static Logger logger = LoggerFactory.getLogger(CosmosAsyncClientTest.class.getSimpleName()); + protected static final int SUITE_SETUP_TIMEOUT = 120000; + private final static AtomicInteger instancesUsed = new AtomicInteger(0); private final CosmosClientBuilder clientBuilder; private String testName; + private volatile Map activeClientsAtBegin = new HashMap<>(); public CosmosAsyncClientTest() { this(new CosmosClientBuilder()); @@ -32,6 +49,81 @@ public CosmosAsyncClientTest(CosmosClientBuilder clientBuilder) { this.clientBuilder = clientBuilder; } + @BeforeClass(groups = {"thinclient", "fast", "long", "direct", "multi-region", "multi-master", "flaky-multi-master", "emulator", + "emulator-vnext", "split", "query", "cfp-split", "circuit-breaker-misc-gateway", "circuit-breaker-misc-direct", + "circuit-breaker-read-all-read-many", "fi-multi-master", "long-emulator", "fi-thinclient-multi-region", "fi-thinclient-multi-master"}, timeOut = SUITE_SETUP_TIMEOUT, alwaysRun = true) + public void beforeClassSetupLeakDetection() { + if (instancesUsed.getAndIncrement() == 0) { + this.activeClientsAtBegin = RxDocumentClientImpl.getActiveClientsSnapshot(); + this.logMemoryUsage("BEFORE"); + } + } + + private void logMemoryUsage(String name) { + long pooledDirectBytes = PooledByteBufAllocator.DEFAULT.metric() + .directArenas().stream() + .mapToLong(io.netty.buffer.PoolArenaMetric::numActiveBytes) + .sum(); + + long used = PlatformDependent.usedDirectMemory(); + long max = PlatformDependent.maxDirectMemory(); + logger.info("MEMORY USAGE: {}:{}", this.getClass().getCanonicalName(), name); + logger.info("Netty Direct Memory: {}/{}/{} bytes", used, pooledDirectBytes, max); + for (BufferPoolMXBean pool : ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class)) { + logger.info("Pool {}: used={} bytes, capacity={} bytes, count={}", + pool.getName(), pool.getMemoryUsed(), pool.getTotalCapacity(), pool.getCount()); + } + } + + @AfterClass(groups = {"thinclient", "fast", "long", "direct", "multi-region", "multi-master", "flaky-multi-master", "emulator", + "emulator-vnext", "split", "query", "cfp-split", "circuit-breaker-misc-gateway", "circuit-breaker-misc-direct", + "circuit-breaker-read-all-read-many", "fi-multi-master", "long-emulator", "fi-thinclient-multi-region", "fi-thinclient-multi-master"}, timeOut = SUITE_SETUP_TIMEOUT, alwaysRun = true) + public void afterClassSetupLeakDetection() { + if (instancesUsed.decrementAndGet() == 0) { + Map leakedClientSnapshotNow = RxDocumentClientImpl.getActiveClientsSnapshot(); + StringBuilder sb = new StringBuilder(); + Map leakedClientSnapshotAtBegin = activeClientsAtBegin; + + for (Integer clientId : leakedClientSnapshotNow.keySet()) { + if (!leakedClientSnapshotAtBegin.containsKey(clientId)) { + // this client was leaked in this class + sb + .append("CosmosClient [") + .append(clientId) + .append("] leaked. Callstack of initialization:\n") + .append(leakedClientSnapshotNow.get(clientId)) + .append("\n\n"); + } + } + + if (sb.length() > 0) { + String msg = "\"COSMOS CLIENT LEAKS detected in test class: " + + this.getClass().getCanonicalName() + + "\n\n" + + sb; + + logger.error(msg); + // fail(msg); + } + + List nettyLeaks = CosmosNettyLeakDetectorFactory.resetIdentifiedLeaks(); + if (nettyLeaks.size() > 0) { + sb.append("\n"); + for (String leak : nettyLeaks) { + sb.append(leak).append("\n"); + } + + String msg = "\"NETTY LEAKS detected in test class: " + + this.getClass().getCanonicalName() + + sb; + + logger.error(msg); + // fail(msg); + } + this.logMemoryUsage("AFTER"); + } + } + public final CosmosClientBuilder getClientBuilder() { return this.clientBuilder; } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/DocumentClientTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/DocumentClientTest.java index e82dd051b1a5..1050c0412f94 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/DocumentClientTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/DocumentClientTest.java @@ -4,17 +4,33 @@ package com.azure.cosmos; import com.azure.cosmos.implementation.AsyncDocumentClient; +import com.azure.cosmos.implementation.RxDocumentClientImpl; import com.azure.cosmos.implementation.guava27.Strings; +import io.netty.buffer.PooledByteBufAllocator; +import io.netty.util.internal.PlatformDependent; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.testng.ITest; +import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; +import java.lang.management.BufferPoolMXBean; +import java.lang.management.ManagementFactory; import java.lang.reflect.Method; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; public abstract class DocumentClientTest implements ITest { - + protected static Logger logger = LoggerFactory.getLogger(DocumentClientTest.class.getSimpleName()); + protected static final int SUITE_SETUP_TIMEOUT = 120000; + private final static AtomicInteger instancesUsed = new AtomicInteger(0); private final AsyncDocumentClient.Builder clientBuilder; private String testName; + private volatile Map activeClientsAtBegin = new HashMap<>(); public DocumentClientTest() { this(new AsyncDocumentClient.Builder()); @@ -28,6 +44,80 @@ public final AsyncDocumentClient.Builder clientBuilder() { return this.clientBuilder; } + @BeforeClass(groups = {"fast", "long", "direct", "multi-region", "multi-master", "flaky-multi-master", "emulator", + "split", "query", "cfp-split", "long-emulator"}, timeOut = SUITE_SETUP_TIMEOUT) + + public void beforeClassSetupLeakDetection() { + if (instancesUsed.getAndIncrement() == 0) { + this.activeClientsAtBegin = RxDocumentClientImpl.getActiveClientsSnapshot(); + this.logMemoryUsage("BEFORE"); + } + } + + private void logMemoryUsage(String name) { + long pooledDirectBytes = PooledByteBufAllocator.DEFAULT.metric() + .directArenas().stream() + .mapToLong(io.netty.buffer.PoolArenaMetric::numActiveBytes) + .sum(); + + long used = PlatformDependent.usedDirectMemory(); + long max = PlatformDependent.maxDirectMemory(); + logger.info("MEMORY USAGE: {}:{}", this.getClass().getCanonicalName(), name); + logger.info("Netty Direct Memory: {}/{}/{} bytes", used, pooledDirectBytes, max); + for (BufferPoolMXBean pool : ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class)) { + logger.info("Pool {}: used={} bytes, capacity={} bytes, count={}", + pool.getName(), pool.getMemoryUsed(), pool.getTotalCapacity(), pool.getCount()); + } + } + + @AfterClass(groups = {"fast", "long", "direct", "multi-region", "multi-master", "flaky-multi-master", "emulator", + "split", "query", "cfp-split", "long-emulator"}, timeOut = SUITE_SETUP_TIMEOUT) + public void afterClassSetupLeakDetection() { + if (instancesUsed.decrementAndGet() == 0) { + Map leakedClientSnapshotNow = RxDocumentClientImpl.getActiveClientsSnapshot(); + StringBuilder sb = new StringBuilder(); + Map leakedClientSnapshotAtBegin = activeClientsAtBegin; + + for (Integer clientId : leakedClientSnapshotNow.keySet()) { + if (!leakedClientSnapshotAtBegin.containsKey(clientId)) { + // this client was leaked in this class + sb + .append("CosmosClient [") + .append(clientId) + .append("] leaked. Callstack of initialization:\n") + .append(leakedClientSnapshotNow.get(clientId)) + .append("\n\n"); + } + } + + if (sb.length() > 0) { + String msg = "\"COSMOS CLIENT LEAKS detected in test class: " + + this.getClass().getCanonicalName() + + "\n\n" + + sb; + + logger.error(msg); + // fail(msg); + } + + List nettyLeaks = CosmosNettyLeakDetectorFactory.resetIdentifiedLeaks(); + if (nettyLeaks.size() > 0) { + sb.append("\n"); + for (String leak : nettyLeaks) { + sb.append(leak).append("\n"); + } + + String msg = "\"NETTY LEAKS detected in test class: " + + this.getClass().getCanonicalName() + + sb; + + logger.error(msg); + // fail(msg); + } + this.logMemoryUsage("AFTER"); + } + } + @Override public final String getTestName() { return this.testName; diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/RegionScopedSessionContainerTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/RegionScopedSessionContainerTest.java index e3cdd0fb16ea..fc537e091437 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/RegionScopedSessionContainerTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/RegionScopedSessionContainerTest.java @@ -18,6 +18,8 @@ import org.apache.commons.lang3.tuple.Pair; import org.mockito.Mockito; import org.mockito.internal.util.collections.Sets; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -31,7 +33,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; -import static com.azure.cosmos.implementation.TestSuiteBase.logger; import static com.azure.cosmos.implementation.TestUtils.mockDiagnosticsClientContext; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.fail; @@ -40,6 +41,7 @@ * Tests for {@link RegionScopedSessionContainer} */ public class RegionScopedSessionContainerTest { + protected static Logger logger = LoggerFactory.getLogger(RegionScopedSessionContainerTest.class.getSimpleName()); private final static URI DefaultEndpoint = createUrl("https://default.documents.azure.com"); private final static Pair LocationEastUsEndpointToLocationPair = Pair.of(createUrl("https://contoso-east-us.documents.azure.com"), "eastus"); diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestSuiteBase.java index 588658444b08..3da097f3213c 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestSuiteBase.java @@ -36,12 +36,8 @@ import org.apache.commons.lang3.StringUtils; import org.mockito.Mockito; import org.mockito.stubbing.Answer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.testng.ITestContext; -import org.testng.annotations.AfterClass; import org.testng.annotations.AfterSuite; -import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeSuite; import org.testng.annotations.DataProvider; import org.testng.annotations.Listeners; @@ -51,28 +47,23 @@ import java.time.Duration; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Fail.fail; import static org.mockito.Mockito.doAnswer; @Listeners({TestNGLogListener.class}) public class TestSuiteBase extends DocumentClientTest { private static final int DEFAULT_BULK_INSERT_CONCURRENCY_LEVEL = 500; private static final ObjectMapper objectMapper = new ObjectMapper(); - protected static Logger logger = LoggerFactory.getLogger(TestSuiteBase.class.getSimpleName()); protected static final int TIMEOUT = 40000; protected static final int FEED_TIMEOUT = 40000; protected static final int SETUP_TIMEOUT = 60000; protected static final int SHUTDOWN_TIMEOUT = 12000; - protected static final int SUITE_SETUP_TIMEOUT = 120000; protected static final int SUITE_SHUTDOWN_TIMEOUT = 60000; protected static final int WAIT_REPLICA_CATCH_UP_IN_MILLIS = 4000; @@ -88,8 +79,6 @@ public class TestSuiteBase extends DocumentClientTest { protected static DocumentCollection SHARED_SINGLE_PARTITION_COLLECTION; protected static DocumentCollection SHARED_MULTI_PARTITION_COLLECTION_WITH_COMPOSITE_AND_SPATIAL_INDEXES; - private volatile Map activeClientsAtBegin = new HashMap<>(); - private static ImmutableList immutableListOrNull(List list) { return list != null ? ImmutableList.copyOf(list) : null; } @@ -151,57 +140,6 @@ public Mono> deleteDatabase(String id) { } } - @BeforeClass(groups = {"fast", "long", "direct", "multi-region", "multi-master", "flaky-multi-master", "emulator", - "split", "query", "cfp-split", "long-emulator"}, timeOut = SUITE_SETUP_TIMEOUT) - - public void beforeClassSetupLeakDetection() { - this.activeClientsAtBegin = RxDocumentClientImpl.getActiveClientsSnapshot(); - } - - @AfterClass(groups = {"fast", "long", "direct", "multi-region", "multi-master", "flaky-multi-master", "emulator", - "split", "query", "cfp-split", "long-emulator"}, timeOut = SUITE_SETUP_TIMEOUT) - public void afterClassSetupLeakDetection() { - - Map leakedClientSnapshotNow = RxDocumentClientImpl.getActiveClientsSnapshot(); - StringBuilder sb = new StringBuilder(); - Map leakedClientSnapshotAtBegin = activeClientsAtBegin; - - for (Integer clientId : leakedClientSnapshotNow.keySet()) { - if (!leakedClientSnapshotAtBegin.containsKey(clientId)) { - // this client was leaked in this class - sb - .append("CosmosClient [") - .append(clientId) - .append("] leaked. Callstack of initialization:\n") - .append(leakedClientSnapshotNow.get(clientId)); - } - } - - if (sb.length() > 0) { - String msg = "\"COSMOS CLIENT LEAKS detected in test class: " - + this.getClass().getCanonicalName() - + sb; - - logger.error(msg); - // fail(msg); - } - - List nettyLeaks = CosmosNettyLeakDetectorFactory.resetIdentifiedLeaks(); - if (nettyLeaks.size() > 0) { - sb.append("\n"); - for (String leak : nettyLeaks) { - sb.append(leak).append("\n"); - } - - String msg = "\"NETTY LEAKS detected in test class: " - + this.getClass().getCanonicalName() - + sb; - - logger.error(msg); - // fail(msg); - } - } - @BeforeSuite(groups = {"fast", "long", "direct", "multi-region", "multi-master", "flaky-multi-master", "emulator", "split", "query", "cfp-split", "long-emulator"}, timeOut = SUITE_SETUP_TIMEOUT) public void beforeSuite() { diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java index c8217f793afc..d17e872db25e 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java @@ -33,7 +33,6 @@ import com.azure.cosmos.implementation.InternalObjectNode; import com.azure.cosmos.implementation.PathParser; import com.azure.cosmos.implementation.Resource; -import com.azure.cosmos.implementation.RxDocumentClientImpl; import com.azure.cosmos.implementation.TestConfigurations; import com.azure.cosmos.implementation.Utils; import com.azure.cosmos.implementation.directconnectivity.Protocol; @@ -71,12 +70,8 @@ import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.mockito.stubbing.Answer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.testng.ITestContext; -import org.testng.annotations.AfterClass; import org.testng.annotations.AfterSuite; -import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeSuite; import org.testng.annotations.DataProvider; import org.testng.annotations.Listeners; @@ -89,9 +84,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -99,7 +92,6 @@ import static com.azure.cosmos.BridgeInternal.extractConfigs; import static com.azure.cosmos.BridgeInternal.injectConfigs; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Fail.fail; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.spy; @@ -109,13 +101,11 @@ public class TestSuiteBase extends CosmosAsyncClientTest { private static final int DEFAULT_BULK_INSERT_CONCURRENCY_LEVEL = 500; private static final ObjectMapper objectMapper = new ObjectMapper(); - protected static Logger logger = LoggerFactory.getLogger(TestSuiteBase.class.getSimpleName()); protected static final int TIMEOUT = 40000; protected static final int FEED_TIMEOUT = 40000; protected static final int SETUP_TIMEOUT = 60000; protected static final int SHUTDOWN_TIMEOUT = 24000; - protected static final int SUITE_SETUP_TIMEOUT = 120000; protected static final int SUITE_SHUTDOWN_TIMEOUT = 60000; protected static final int WAIT_REPLICA_CATCH_UP_IN_MILLIS = 4000; @@ -129,8 +119,6 @@ public class TestSuiteBase extends CosmosAsyncClientTest { protected int subscriberValidationTimeout = TIMEOUT; - private volatile Map activeClientsAtBegin = new HashMap<>(); - private static CosmosAsyncDatabase SHARED_DATABASE; private static CosmosAsyncContainer SHARED_MULTI_PARTITION_COLLECTION_WITH_ID_AS_PARTITION_KEY; private static CosmosAsyncContainer SHARED_MULTI_PARTITION_COLLECTION; @@ -215,58 +203,6 @@ public CosmosAsyncDatabase getDatabase(String id) { } } - @BeforeClass(groups = {"thinclient", "fast", "long", "direct", "multi-region", "multi-master", "flaky-multi-master", "emulator", - "emulator-vnext", "split", "query", "cfp-split", "circuit-breaker-misc-gateway", "circuit-breaker-misc-direct", - "circuit-breaker-read-all-read-many", "fi-multi-master", "long-emulator", "fi-thinclient-multi-region", "fi-thinclient-multi-master"}, timeOut = SUITE_SETUP_TIMEOUT) - public void beforeClassSetupLeakDetection() { - this.activeClientsAtBegin = RxDocumentClientImpl.getActiveClientsSnapshot(); - } - - @AfterClass(groups = {"thinclient", "fast", "long", "direct", "multi-region", "multi-master", "flaky-multi-master", "emulator", - "emulator-vnext", "split", "query", "cfp-split", "circuit-breaker-misc-gateway", "circuit-breaker-misc-direct", - "circuit-breaker-read-all-read-many", "fi-multi-master", "long-emulator", "fi-thinclient-multi-region", "fi-thinclient-multi-master"}, timeOut = SUITE_SETUP_TIMEOUT) - public void afterClassSetupLeakDetection() { - - Map leakedClientSnapshotNow = RxDocumentClientImpl.getActiveClientsSnapshot(); - StringBuilder sb = new StringBuilder(); - Map leakedClientSnapshotAtBegin = activeClientsAtBegin; - - for (Integer clientId : leakedClientSnapshotNow.keySet()) { - if (!leakedClientSnapshotAtBegin.containsKey(clientId)) { - // this client was leaked in this class - sb - .append("CosmosClient [") - .append(clientId) - .append("] leaked. Callstack of initialization:\n") - .append(leakedClientSnapshotNow.get(clientId)); - } - } - - if (sb.length() > 0) { - String msg = "\"COSMOS CLIENT LEAKS detected in test class: " - + this.getClass().getCanonicalName() - + sb; - - logger.error(msg); - // fail(msg); - } - - List nettyLeaks = CosmosNettyLeakDetectorFactory.resetIdentifiedLeaks(); - if (nettyLeaks.size() > 0) { - sb.append("\n"); - for (String leak : nettyLeaks) { - sb.append(leak).append("\n"); - } - - String msg = "\"NETTY LEAKS detected in test class: " - + this.getClass().getCanonicalName() - + sb; - - logger.error(msg); - // fail(msg); - } - } - @BeforeSuite(groups = {"thinclient", "fast", "long", "direct", "multi-region", "multi-master", "flaky-multi-master", "emulator", "emulator-vnext", "split", "query", "cfp-split", "circuit-breaker-misc-gateway", "circuit-breaker-misc-direct", "circuit-breaker-read-all-read-many", "fi-multi-master", "long-emulator", "fi-thinclient-multi-region", "fi-thinclient-multi-master"}, timeOut = SUITE_SETUP_TIMEOUT) From 1046b1d993e92ac77a84857d97bc1cb7aee1d8be Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Mon, 10 Nov 2025 13:21:53 +0000 Subject: [PATCH 19/65] Cleaning-up dummy QueryFeedRangeState properly --- .../EndToEndTimeOutValidationTests.java | 190 ++++++++++------ .../com/azure/cosmos/UserAgentSuffixTest.java | 85 +++---- .../implementation/ConsistencyTests2.java | 4 +- .../DocumentQuerySpyWireContentTest.java | 38 ++-- .../RequestHeadersSpyWireTest.java | 55 +++-- .../cosmos/implementation/SessionTest.java | 214 +++++++++--------- .../cosmos/implementation/TestSuiteBase.java | 84 +++++-- .../DCDocumentCrudTest.java | 28 ++- .../com/azure/cosmos/rx/BackPressureTest.java | 34 +-- .../com/azure/cosmos/rx/OfferQueryTest.java | 60 +++-- .../azure/cosmos/rx/OfferReadReplaceTest.java | 73 +++--- .../azure/cosmos/rx/ReadFeedOffersTest.java | 17 +- .../azure/cosmos/rx/ResourceTokenTest.java | 13 +- .../com/azure/cosmos/rx/TestSuiteBase.java | 7 + .../implementation/AsyncDocumentClient.java | 9 +- .../implementation/FeedOperationState.java | 4 + .../implementation/RxGatewayStoreModel.java | 69 +++--- 17 files changed, 595 insertions(+), 389 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/EndToEndTimeOutValidationTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/EndToEndTimeOutValidationTests.java index 1277992b9b8d..e15d6a25ca0c 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/EndToEndTimeOutValidationTests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/EndToEndTimeOutValidationTests.java @@ -55,12 +55,7 @@ public EndToEndTimeOutValidationTests(CosmosClientBuilder clientBuilder) { .build(); } - @BeforeClass(groups = {"fast"}, timeOut = SETUP_TIMEOUT * 100) - public void beforeClass() throws Exception { - initializeClient(null); - } - - public void initializeClient(CosmosEndToEndOperationLatencyPolicyConfig e2eDefaultConfig) { + public CosmosAsyncClient initializeClient(CosmosEndToEndOperationLatencyPolicyConfig e2eDefaultConfig) { CosmosAsyncClient client = this .getClientBuilder() .endToEndOperationLatencyPolicyConfig(e2eDefaultConfig) @@ -69,6 +64,8 @@ public void initializeClient(CosmosEndToEndOperationLatencyPolicyConfig e2eDefau truncateCollection(createdContainer); createdDocuments.addAll(this.insertDocuments(DEFAULT_NUM_DOCUMENTS, null, createdContainer)); + + return client; } @Test(groups = {"fast"}, timeOut = 10000L, retryAnalyzer = FlakyTestRetryAnalyzer.class) @@ -77,17 +74,24 @@ public void readItemWithEndToEndTimeoutPolicyInOptionsShouldTimeoutWithClientCon throw new SkipException("Failure injection only supported for DIRECT mode"); } - initializeClient(endToEndOperationLatencyPolicyConfig); + CosmosAsyncClient cosmosClient = initializeClient(endToEndOperationLatencyPolicyConfig); + FaultInjectionRule rule = null; + try { - CosmosItemRequestOptions options = new CosmosItemRequestOptions(); - TestObject itemToRead = createdDocuments.get(random.nextInt(createdDocuments.size())); - FaultInjectionRule rule = injectFailure(createdContainer, FaultInjectionOperationType.READ_ITEM, null); + CosmosItemRequestOptions options = new CosmosItemRequestOptions(); + TestObject itemToRead = createdDocuments.get(random.nextInt(createdDocuments.size())); + rule = injectFailure(createdContainer, FaultInjectionOperationType.READ_ITEM, null); - Mono> cosmosItemResponseMono = - createdContainer.readItem(itemToRead.id, new PartitionKey(itemToRead.mypk), options, TestObject.class); + Mono> cosmosItemResponseMono = + createdContainer.readItem(itemToRead.id, new PartitionKey(itemToRead.mypk), options, TestObject.class); - verifyExpectError(cosmosItemResponseMono); - rule.disable(); + verifyExpectError(cosmosItemResponseMono); + } finally { + if (rule != null) { + rule.disable(); + } + safeClose(cosmosClient); + } } @Test(groups = {"fast"}, timeOut = 10000L, retryAnalyzer = FlakyTestRetryAnalyzer.class) @@ -100,7 +104,7 @@ public void readItemWithEndToEndTimeoutPolicyInOptionsShouldTimeoutEvenWhenDisab Configs.DEFAULT_E2E_FOR_NON_POINT_DISABLED, "true"); - initializeClient(endToEndOperationLatencyPolicyConfig); + CosmosAsyncClient cosmosClient = initializeClient(endToEndOperationLatencyPolicyConfig); FaultInjectionRule rule = null; try { @@ -119,6 +123,7 @@ public void readItemWithEndToEndTimeoutPolicyInOptionsShouldTimeoutEvenWhenDisab } System.clearProperty(Configs.DEFAULT_E2E_FOR_NON_POINT_DISABLED); + safeClose(cosmosClient); } } @@ -128,16 +133,24 @@ public void createItemWithEndToEndTimeoutPolicyInOptionsShouldTimeout() { throw new SkipException("Failure injection only supported for DIRECT mode"); } - CosmosItemRequestOptions options = new CosmosItemRequestOptions(); - options.setCosmosEndToEndOperationLatencyPolicyConfig(endToEndOperationLatencyPolicyConfig); + CosmosAsyncClient cosmosClient = initializeClient(endToEndOperationLatencyPolicyConfig); + FaultInjectionRule faultInjectionRule = null; + try { + CosmosItemRequestOptions options = new CosmosItemRequestOptions(); + options.setCosmosEndToEndOperationLatencyPolicyConfig(endToEndOperationLatencyPolicyConfig); - FaultInjectionRule faultInjectionRule = injectFailure(createdContainer, FaultInjectionOperationType.CREATE_ITEM, null); - TestObject inputObject = new TestObject(UUID.randomUUID().toString(), "name123", 1, UUID.randomUUID().toString()); - Mono> cosmosItemResponseMono = - createdContainer.createItem(inputObject, new PartitionKey(inputObject.mypk), options); + faultInjectionRule = injectFailure(createdContainer, FaultInjectionOperationType.CREATE_ITEM, null); + TestObject inputObject = new TestObject(UUID.randomUUID().toString(), "name123", 1, UUID.randomUUID().toString()); + Mono> cosmosItemResponseMono = + createdContainer.createItem(inputObject, new PartitionKey(inputObject.mypk), options); - verifyExpectError(cosmosItemResponseMono); - faultInjectionRule.disable(); + verifyExpectError(cosmosItemResponseMono); + } finally { + if (faultInjectionRule != null) { + faultInjectionRule.disable(); + } + safeClose(cosmosClient); + } } @Test(groups = {"fast"}, timeOut = 10000L, retryAnalyzer = FlakyTestRetryAnalyzer.class) @@ -146,18 +159,26 @@ public void replaceItemWithEndToEndTimeoutPolicyInOptionsShouldTimeout() { throw new SkipException("Failure injection only supported for DIRECT mode"); } - CosmosItemRequestOptions options = new CosmosItemRequestOptions(); - options.setCosmosEndToEndOperationLatencyPolicyConfig(endToEndOperationLatencyPolicyConfig); + CosmosAsyncClient cosmosClient = initializeClient(endToEndOperationLatencyPolicyConfig); + FaultInjectionRule rule = null; + try { + CosmosItemRequestOptions options = new CosmosItemRequestOptions(); + options.setCosmosEndToEndOperationLatencyPolicyConfig(endToEndOperationLatencyPolicyConfig); - TestObject inputObject = new TestObject(UUID.randomUUID().toString(), "name123", 1, UUID.randomUUID().toString()); - createdContainer.createItem(inputObject, new PartitionKey(inputObject.mypk), options).block(); - FaultInjectionRule rule = injectFailure(createdContainer, FaultInjectionOperationType.REPLACE_ITEM, null); - inputObject.setName("replaceName"); - Mono> cosmosItemResponseMono = - createdContainer.replaceItem(inputObject, inputObject.id, new PartitionKey(inputObject.mypk), options); + TestObject inputObject = new TestObject(UUID.randomUUID().toString(), "name123", 1, UUID.randomUUID().toString()); + createdContainer.createItem(inputObject, new PartitionKey(inputObject.mypk), options).block(); + rule = injectFailure(createdContainer, FaultInjectionOperationType.REPLACE_ITEM, null); + inputObject.setName("replaceName"); + Mono> cosmosItemResponseMono = + createdContainer.replaceItem(inputObject, inputObject.id, new PartitionKey(inputObject.mypk), options); - verifyExpectError(cosmosItemResponseMono); - rule.disable(); + verifyExpectError(cosmosItemResponseMono); + } finally { + if (rule != null) { + rule.disable(); + } + safeClose(cosmosClient); + } } @Test(groups = {"fast"}, timeOut = 10000L, retryAnalyzer = FlakyTestRetryAnalyzer.class) @@ -166,16 +187,25 @@ public void upsertItemWithEndToEndTimeoutPolicyInOptionsShouldTimeout() { throw new SkipException("Failure injection only supported for DIRECT mode"); } - CosmosItemRequestOptions options = new CosmosItemRequestOptions(); - options.setCosmosEndToEndOperationLatencyPolicyConfig(endToEndOperationLatencyPolicyConfig); + CosmosAsyncClient cosmosClient = initializeClient(endToEndOperationLatencyPolicyConfig); + FaultInjectionRule rule = null; + try { + CosmosItemRequestOptions options = new CosmosItemRequestOptions(); + options.setCosmosEndToEndOperationLatencyPolicyConfig(endToEndOperationLatencyPolicyConfig); + + rule = injectFailure(createdContainer, FaultInjectionOperationType.UPSERT_ITEM, null); + TestObject inputObject = new TestObject(UUID.randomUUID().toString(), "name123", 1, UUID.randomUUID().toString()); + Mono> cosmosItemResponseMono = + createdContainer.upsertItem(inputObject, new PartitionKey(inputObject.mypk), options); - FaultInjectionRule rule = injectFailure(createdContainer, FaultInjectionOperationType.UPSERT_ITEM, null); - TestObject inputObject = new TestObject(UUID.randomUUID().toString(), "name123", 1, UUID.randomUUID().toString()); - Mono> cosmosItemResponseMono = - createdContainer.upsertItem(inputObject, new PartitionKey(inputObject.mypk), options); + verifyExpectError(cosmosItemResponseMono); + } finally { + if (rule != null) { + rule.disable(); + } - verifyExpectError(cosmosItemResponseMono); - rule.disable(); + safeClose(cosmosClient); + } } static void verifyExpectError(Mono> cosmosItemResponseMono) { @@ -190,27 +220,36 @@ public void queryItemWithEndToEndTimeoutPolicyInOptionsShouldTimeout() { throw new SkipException("Failure injection only supported for DIRECT mode"); } - CosmosEndToEndOperationLatencyPolicyConfig endToEndOperationLatencyPolicyConfig = - new CosmosEndToEndOperationLatencyPolicyConfigBuilder(Duration.ofSeconds(1)) - .build(); + CosmosAsyncClient cosmosClient = initializeClient(endToEndOperationLatencyPolicyConfig); + FaultInjectionRule faultInjectionRule = null; + try { + CosmosEndToEndOperationLatencyPolicyConfig endToEndOperationLatencyPolicyConfig = + new CosmosEndToEndOperationLatencyPolicyConfigBuilder(Duration.ofSeconds(1)) + .build(); - CosmosQueryRequestOptions options = new CosmosQueryRequestOptions(); - options.setCosmosEndToEndOperationLatencyPolicyConfig(endToEndOperationLatencyPolicyConfig); + CosmosQueryRequestOptions options = new CosmosQueryRequestOptions(); + options.setCosmosEndToEndOperationLatencyPolicyConfig(endToEndOperationLatencyPolicyConfig); - createdDocuments.get(random.nextInt(createdDocuments.size())); + createdDocuments.get(random.nextInt(createdDocuments.size())); - String queryText = "select top 1 * from c"; - SqlQuerySpec sqlQuerySpec = new SqlQuerySpec(queryText); + String queryText = "select top 1 * from c"; + SqlQuerySpec sqlQuerySpec = new SqlQuerySpec(queryText); - FaultInjectionRule faultInjectionRule = injectFailure(createdContainer, FaultInjectionOperationType.QUERY_ITEM, null); - CosmosPagedFlux queryPagedFlux = createdContainer.queryItems(sqlQuerySpec, options, TestObject.class); + faultInjectionRule = injectFailure(createdContainer, FaultInjectionOperationType.QUERY_ITEM, null); + CosmosPagedFlux queryPagedFlux = createdContainer.queryItems(sqlQuerySpec, options, TestObject.class); - StepVerifier.create(queryPagedFlux) - .expectErrorMatches(throwable -> throwable instanceof OperationCancelledException - && ((OperationCancelledException) throwable).getSubStatusCode() - == HttpConstants.SubStatusCodes.CLIENT_OPERATION_TIMEOUT) - .verify(); - faultInjectionRule.disable(); + StepVerifier.create(queryPagedFlux) + .expectErrorMatches(throwable -> throwable instanceof OperationCancelledException + && ((OperationCancelledException) throwable).getSubStatusCode() + == HttpConstants.SubStatusCodes.CLIENT_OPERATION_TIMEOUT) + .verify(); + } finally { + if (faultInjectionRule != null) { + faultInjectionRule.disable(); + } + + safeClose(cosmosClient); + } } @Test(groups = {"fast"}, timeOut = 10000L, retryAnalyzer = FlakyTestRetryAnalyzer.class) @@ -223,24 +262,30 @@ public void queryItemWithEndToEndTimeoutPolicyInOptionsShouldTimeoutWithClientCo new CosmosEndToEndOperationLatencyPolicyConfigBuilder(Duration.ofSeconds(1)) .build(); - initializeClient(endToEndOperationLatencyPolicyConfig); - - CosmosQueryRequestOptions options = new CosmosQueryRequestOptions(); + CosmosAsyncClient cosmosClient = initializeClient(endToEndOperationLatencyPolicyConfig); + FaultInjectionRule faultInjectionRule = null; + try { + CosmosQueryRequestOptions options = new CosmosQueryRequestOptions(); - createdDocuments.get(random.nextInt(createdDocuments.size())); + createdDocuments.get(random.nextInt(createdDocuments.size())); - String queryText = "select top 1 * from c"; - SqlQuerySpec sqlQuerySpec = new SqlQuerySpec(queryText); + String queryText = "select top 1 * from c"; + SqlQuerySpec sqlQuerySpec = new SqlQuerySpec(queryText); - FaultInjectionRule faultInjectionRule = injectFailure(createdContainer, FaultInjectionOperationType.QUERY_ITEM, null); - CosmosPagedFlux queryPagedFlux = createdContainer.queryItems(sqlQuerySpec, options, TestObject.class); + faultInjectionRule = injectFailure(createdContainer, FaultInjectionOperationType.QUERY_ITEM, null); + CosmosPagedFlux queryPagedFlux = createdContainer.queryItems(sqlQuerySpec, options, TestObject.class); - StepVerifier.create(queryPagedFlux) - .expectErrorMatches(throwable -> throwable instanceof OperationCancelledException - && ((OperationCancelledException) throwable).getSubStatusCode() - == HttpConstants.SubStatusCodes.CLIENT_OPERATION_TIMEOUT) - .verify(); - faultInjectionRule.disable(); + StepVerifier.create(queryPagedFlux) + .expectErrorMatches(throwable -> throwable instanceof OperationCancelledException + && ((OperationCancelledException) throwable).getSubStatusCode() + == HttpConstants.SubStatusCodes.CLIENT_OPERATION_TIMEOUT) + .verify(); + } finally { + if (faultInjectionRule != null) { + faultInjectionRule.disable(); + } + safeClose(cosmosClient); + } } @Test(groups = {"fast"}, timeOut = 10000L, retryAnalyzer = FlakyTestRetryAnalyzer.class) @@ -257,7 +302,7 @@ public void queryItemWithEndToEndTimeoutPolicyInOptionsShouldNotTimeoutWhenSuppr "isDefaultE2ETimeoutDisabledForNonPointOperations() after setting system property {}", Configs.isDefaultE2ETimeoutDisabledForNonPointOperations()); - initializeClient( + CosmosAsyncClient cosmosClient = initializeClient( new CosmosEndToEndOperationLatencyPolicyConfigBuilder(Duration.ofSeconds(1)) .build() ); @@ -283,6 +328,7 @@ public void queryItemWithEndToEndTimeoutPolicyInOptionsShouldNotTimeoutWhenSuppr } System.clearProperty(Configs.DEFAULT_E2E_FOR_NON_POINT_DISABLED); + safeClose(cosmosClient); } } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/UserAgentSuffixTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/UserAgentSuffixTest.java index d91b97327f80..116eb5b72e0e 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/UserAgentSuffixTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/UserAgentSuffixTest.java @@ -9,6 +9,7 @@ import com.azure.cosmos.implementation.Configs; import com.azure.cosmos.models.CosmosContainerResponse; import com.azure.cosmos.rx.TestSuiteBase; +import org.testng.ITestContext; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Factory; @@ -42,70 +43,74 @@ public void afterClass() { @Test(groups = { "fast", "emulator" }, timeOut = TIMEOUT) public void userAgentSuffixWithoutSpecialCharacter() { - CosmosClient clientWithUserAgentSuffix = getClientBuilder() + try (CosmosClient clientWithUserAgentSuffix = getClientBuilder() .userAgentSuffix("TestUserAgent") - .buildClient(); + .buildClient()) { - CosmosContainerResponse response = - clientWithUserAgentSuffix.getDatabase(this.databaseName).getContainer(this.containerName).read(); + CosmosContainerResponse response = + clientWithUserAgentSuffix.getDatabase(this.databaseName).getContainer(this.containerName).read(); - assertThat(response).isNotNull(); - assertThat(response.getStatusCode()).isEqualTo(200); - assertThat(response.getProperties()).isNotNull(); - assertThat(response.getProperties().getId()).isEqualTo(this.containerName); - assertThat(response.getDiagnostics()).isNotNull(); - validateUserAgentSuffix(response.getDiagnostics().getUserAgent(), "TestUserAgent"); + assertThat(response).isNotNull(); + assertThat(response.getStatusCode()).isEqualTo(200); + assertThat(response.getProperties()).isNotNull(); + assertThat(response.getProperties().getId()).isEqualTo(this.containerName); + assertThat(response.getDiagnostics()).isNotNull(); + validateUserAgentSuffix(response.getDiagnostics().getUserAgent(), "TestUserAgent"); + } } @Test(groups = { "fast", "emulator" }, timeOut = TIMEOUT) public void userAgentSuffixWithSpecialCharacter() { - CosmosClient clientWithUserAgentSuffix = getClientBuilder() + try (CosmosClient clientWithUserAgentSuffix = getClientBuilder() .userAgentSuffix("TéstUserAgent's") - .buildClient(); + .buildClient()) { - CosmosContainerResponse response = - clientWithUserAgentSuffix.getDatabase(this.databaseName).getContainer(this.containerName).read(); + CosmosContainerResponse response = + clientWithUserAgentSuffix.getDatabase(this.databaseName).getContainer(this.containerName).read(); - assertThat(response).isNotNull(); - assertThat(response.getStatusCode()).isEqualTo(200); - assertThat(response.getProperties()).isNotNull(); - assertThat(response.getProperties().getId()).isEqualTo(this.containerName); - assertThat(response.getDiagnostics()).isNotNull(); - validateUserAgentSuffix(response.getDiagnostics().getUserAgent(), "TestUserAgent's"); + assertThat(response).isNotNull(); + assertThat(response.getStatusCode()).isEqualTo(200); + assertThat(response.getProperties()).isNotNull(); + assertThat(response.getProperties().getId()).isEqualTo(this.containerName); + assertThat(response.getDiagnostics()).isNotNull(); + validateUserAgentSuffix(response.getDiagnostics().getUserAgent(), "TestUserAgent's"); + } } @Test(groups = { "fast", "emulator" }, timeOut = TIMEOUT) public void userAgentSuffixWithUnicodeCharacter() { - CosmosClient clientWithUserAgentSuffix = getClientBuilder() + try (CosmosClient clientWithUserAgentSuffix = getClientBuilder() .userAgentSuffix("UnicodeChar鱀InUserAgent") - .buildClient(); + .buildClient()) { - CosmosContainerResponse response = - clientWithUserAgentSuffix.getDatabase(this.databaseName).getContainer(this.containerName).read(); + CosmosContainerResponse response = + clientWithUserAgentSuffix.getDatabase(this.databaseName).getContainer(this.containerName).read(); - assertThat(response).isNotNull(); - assertThat(response.getStatusCode()).isEqualTo(200); - assertThat(response.getProperties()).isNotNull(); - assertThat(response.getProperties().getId()).isEqualTo(this.containerName); - assertThat(response.getDiagnostics()).isNotNull(); - validateUserAgentSuffix(response.getDiagnostics().getUserAgent(), "UnicodeChar_InUserAgent"); + assertThat(response).isNotNull(); + assertThat(response.getStatusCode()).isEqualTo(200); + assertThat(response.getProperties()).isNotNull(); + assertThat(response.getProperties().getId()).isEqualTo(this.containerName); + assertThat(response.getDiagnostics()).isNotNull(); + validateUserAgentSuffix(response.getDiagnostics().getUserAgent(), "UnicodeChar_InUserAgent"); + } } @Test(groups = { "fast", "emulator" }, timeOut = TIMEOUT) public void userAgentSuffixWithWhitespaceAndAsciiSpecialChars() { - CosmosClient clientWithUserAgentSuffix = getClientBuilder() + try (CosmosClient clientWithUserAgentSuffix = getClientBuilder() .userAgentSuffix("UserAgent with space$%_^()*&") - .buildClient(); + .buildClient()) { - CosmosContainerResponse response = - clientWithUserAgentSuffix.getDatabase(this.databaseName).getContainer(this.containerName).read(); + CosmosContainerResponse response = + clientWithUserAgentSuffix.getDatabase(this.databaseName).getContainer(this.containerName).read(); - assertThat(response).isNotNull(); - assertThat(response.getStatusCode()).isEqualTo(200); - assertThat(response.getProperties()).isNotNull(); - assertThat(response.getProperties().getId()).isEqualTo(this.containerName); - assertThat(response.getDiagnostics()).isNotNull(); - validateUserAgentSuffix(response.getDiagnostics().getUserAgent(), "UserAgent with space$%_^()*&"); + assertThat(response).isNotNull(); + assertThat(response.getStatusCode()).isEqualTo(200); + assertThat(response.getProperties()).isNotNull(); + assertThat(response.getProperties().getId()).isEqualTo(this.containerName); + assertThat(response.getDiagnostics()).isNotNull(); + validateUserAgentSuffix(response.getDiagnostics().getUserAgent(), "UserAgent with space$%_^()*&"); + } } private void validateUserAgentSuffix(String actualUserAgent, String expectedUserAgentSuffix) { diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConsistencyTests2.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConsistencyTests2.java index e13377e64999..38d8c941efa7 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConsistencyTests2.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConsistencyTests2.java @@ -233,6 +233,7 @@ public void validateNoChargeOnFailedSessionRead() throws Exception { new CosmosClientTelemetryConfig() .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) .build(); + QueryFeedOperationState dummyState = null; try { // CREATE collection DocumentCollection parentResource = writeClient.createCollection(createdDatabase.getSelfLink(), @@ -250,7 +251,7 @@ public void validateNoChargeOnFailedSessionRead() throws Exception { cosmosQueryRequestOptions.setPartitionKey(new PartitionKey(PartitionKeyInternal.Empty.toJson())); cosmosQueryRequestOptions.setSessionToken(token); - QueryFeedOperationState dummyState = TestUtils.createDummyQueryFeedOperationState( + dummyState = TestUtils.createDummyQueryFeedOperationState( ResourceType.Document, OperationType.ReadFeed, cosmosQueryRequestOptions, @@ -262,6 +263,7 @@ public void validateNoChargeOnFailedSessionRead() throws Exception { parentResource.getSelfLink(), dummyState, Document.class); validateQueryFailure(feedObservable, validator); } finally { + safeClose(dummyState); safeClose(writeClient); safeClose(readSecondaryClient); } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/DocumentQuerySpyWireContentTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/DocumentQuerySpyWireContentTest.java index 23e7d569a4b1..fcbee5b7f3e9 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/DocumentQuerySpyWireContentTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/DocumentQuerySpyWireContentTest.java @@ -95,22 +95,28 @@ public void queryWithContinuationTokenLimit(CosmosQueryRequestOptions options, S client.clearCapturedRequests(); - Flux> queryObservable = client + QueryFeedOperationState dummyState = TestUtils + .createDummyQueryFeedOperationState(ResourceType.Document, OperationType.Query, options, client); + try { + Flux> queryObservable = client .queryDocuments( collectionLink, query, - TestUtils.createDummyQueryFeedOperationState(ResourceType.Document, OperationType.Query, options, client), + dummyState, Document.class); - List results = queryObservable.flatMap(p -> Flux.fromIterable(p.getResults())) - .collectList().block(); + List results = queryObservable.flatMap(p -> Flux.fromIterable(p.getResults())) + .collectList().block(); - assertThat(results.size()).describedAs("total results").isGreaterThanOrEqualTo(1); + assertThat(results.size()).describedAs("total results").isGreaterThanOrEqualTo(1); - List requests = client.getCapturedRequests(); + List requests = client.getCapturedRequests(); - for(HttpRequest req: requests) { - validateRequestHasContinuationTokenLimit(req, options.getResponseContinuationTokenLimitInKb()); + for (HttpRequest req : requests) { + validateRequestHasContinuationTokenLimit(req, options.getResponseContinuationTokenLimitInKb()); + } + } finally { + safeClose(dummyState); } } @@ -171,14 +177,18 @@ public void before_DocumentQuerySpyWireContentTest() throws Exception { options, client ); + try { - // do the query once to ensure the collection is cached. - client.queryDocuments(getMultiPartitionCollectionLink(), "select * from root", state, Document.class) - .then().block(); + // do the query once to ensure the collection is cached. + client.queryDocuments(getMultiPartitionCollectionLink(), "select * from root", state, Document.class) + .then().block(); - // do the query once to ensure the collection is cached. - client.queryDocuments(getSinglePartitionCollectionLink(), "select * from root", state, Document.class) - .then().block(); + // do the query once to ensure the collection is cached. + client.queryDocuments(getSinglePartitionCollectionLink(), "select * from root", state, Document.class) + .then().block(); + } finally { + safeClose(state); + } } @AfterClass(groups = { "fast" }, timeOut = SHUTDOWN_TIMEOUT, alwaysRun = true) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/RequestHeadersSpyWireTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/RequestHeadersSpyWireTest.java index 435cf569c90b..b9cbde1fc7fd 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/RequestHeadersSpyWireTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/RequestHeadersSpyWireTest.java @@ -137,15 +137,21 @@ public void queryWithMaxIntegratedCacheStaleness(CosmosQueryRequestOptions optio client.clearCapturedRequests(); - client.queryDocuments( - collectionLink, - query, - TestUtils.createDummyQueryFeedOperationState(ResourceType.Document, OperationType.Query, options, client), - Document.class).blockLast(); - - List requests = client.getCapturedRequests(); - for (HttpRequest httpRequest : requests) { - validateRequestHasDedicatedGatewayHeaders(httpRequest, options.getDedicatedGatewayRequestOptions()); + QueryFeedOperationState dummyState = TestUtils + .createDummyQueryFeedOperationState(ResourceType.Document, OperationType.Query, options, client); + try { + client.queryDocuments( + collectionLink, + query, + dummyState, + Document.class).blockLast(); + + List requests = client.getCapturedRequests(); + for (HttpRequest httpRequest : requests) { + validateRequestHasDedicatedGatewayHeaders(httpRequest, options.getDedicatedGatewayRequestOptions()); + } + } finally { + safeClose(dummyState); } } @@ -168,11 +174,16 @@ public void queryWithMaxIntegratedCacheStalenessInNanoseconds() { client ); - assertThatThrownBy(() -> client - .queryDocuments(collectionLink, query, state, Document.class) - .blockLast()) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("MaxIntegratedCacheStaleness granularity is milliseconds"); + try { + + assertThatThrownBy(() -> client + .queryDocuments(collectionLink, query, state, Document.class) + .blockLast()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("MaxIntegratedCacheStaleness granularity is milliseconds"); + } finally { + safeClose(state); + } } @Test(groups = { "fast" }, timeOut = TIMEOUT) @@ -192,13 +203,17 @@ public void queryWithMaxIntegratedCacheStalenessInNegative() { client ); - client.clearCapturedRequests(); + try { + client.clearCapturedRequests(); - assertThatThrownBy(() -> client - .queryDocuments(collectionLink, query, state, Document.class) - .blockLast()) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("MaxIntegratedCacheStaleness duration cannot be negative"); + assertThatThrownBy(() -> client + .queryDocuments(collectionLink, query, state, Document.class) + .blockLast()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("MaxIntegratedCacheStaleness duration cannot be negative"); + } finally { + safeClose(state); + } } @Test(dataProvider = "maxIntegratedCacheStalenessDurationProviderItemOptions", groups = { "fast" }, timeOut = diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/SessionTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/SessionTest.java index 888ed2687825..f1482ac5accf 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/SessionTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/SessionTest.java @@ -203,119 +203,131 @@ public void partitionedSessionToken(boolean isNameBased) throws NoSuchMethodExce spyClient ); - spyClient.queryDocuments(getCollectionLink(isNameBased), query, dummyState, Document.class).blockFirst(); - assertThat(getSessionTokensInRequests()).hasSize(1); - assertThat(getSessionTokensInRequests().get(0)).isNotEmpty(); - assertThat(getSessionTokensInRequests().get(0)).doesNotContain(","); // making sure we have only one scope session token + try { - // Session token validation for cross partition query - spyClient.clearCapturedRequests(); - queryRequestOptions = new CosmosQueryRequestOptions(); - dummyState = TestUtils.createDummyQueryFeedOperationState( - ResourceType.Document, - OperationType.Query, - queryRequestOptions, - spyClient - ); - spyClient.queryDocuments(getCollectionLink(isNameBased), query, dummyState, Document.class).blockFirst(); - assertThat(getSessionTokensInRequests().size()).isGreaterThanOrEqualTo(1); - assertThat(getSessionTokensInRequests().get(0)).isNotEmpty(); - assertThat(getSessionTokensInRequests().get(0)).doesNotContain(","); // making sure we have only one scope session token + spyClient.queryDocuments(getCollectionLink(isNameBased), query, dummyState, Document.class).blockFirst(); + assertThat(getSessionTokensInRequests()).hasSize(1); + assertThat(getSessionTokensInRequests().get(0)).isNotEmpty(); + assertThat(getSessionTokensInRequests().get(0)).doesNotContain(","); // making sure we have only one scope session token - // Session token validation for feed ranges query - spyClient.clearCapturedRequests(); - List feedRanges = spyClient.getFeedRanges(getCollectionLink(isNameBased), true).block(); - queryRequestOptions = new CosmosQueryRequestOptions(); - queryRequestOptions.setFeedRange(feedRanges.get(0)); - dummyState = TestUtils.createDummyQueryFeedOperationState( - ResourceType.Document, - OperationType.Query, - queryRequestOptions, - spyClient - ); - spyClient.queryDocuments(getCollectionLink(isNameBased), query, dummyState, Document.class).blockFirst(); - assertThat(getSessionTokensInRequests().size()).isGreaterThanOrEqualTo(1); - assertThat(getSessionTokensInRequests().get(0)).isNotEmpty(); - assertThat(getSessionTokensInRequests().get(0)).doesNotContain(","); // making sure we have only one scope session token + // Session token validation for cross partition query + spyClient.clearCapturedRequests(); + queryRequestOptions = new CosmosQueryRequestOptions(); + safeClose(dummyState); + dummyState = TestUtils.createDummyQueryFeedOperationState( + ResourceType.Document, + OperationType.Query, + queryRequestOptions, + spyClient + ); + spyClient.queryDocuments(getCollectionLink(isNameBased), query, dummyState, Document.class).blockFirst(); + assertThat(getSessionTokensInRequests().size()).isGreaterThanOrEqualTo(1); + assertThat(getSessionTokensInRequests().get(0)).isNotEmpty(); + assertThat(getSessionTokensInRequests().get(0)).doesNotContain(","); // making sure we have only one scope session token - // Session token validation for readAll with partition query - spyClient.clearCapturedRequests(); - queryRequestOptions = new CosmosQueryRequestOptions(); - dummyState = TestUtils.createDummyQueryFeedOperationState( - ResourceType.Document, - OperationType.ReadFeed, - queryRequestOptions, - spyClient - ); - spyClient.readAllDocuments( - getCollectionLink(isNameBased), - new PartitionKey(documentCreated.getId()), - dummyState, - Document.class).blockFirst(); - assertThat(getSessionTokensInRequests().size()).isEqualTo(1); - assertThat(getSessionTokensInRequests().get(0)).isNotEmpty(); - assertThat(getSessionTokensInRequests().get(0)).doesNotContain(","); // making sure we have only one scope session token + // Session token validation for feed ranges query + spyClient.clearCapturedRequests(); + List feedRanges = spyClient.getFeedRanges(getCollectionLink(isNameBased), true).block(); + queryRequestOptions = new CosmosQueryRequestOptions(); + queryRequestOptions.setFeedRange(feedRanges.get(0)); + safeClose(dummyState); + dummyState = TestUtils.createDummyQueryFeedOperationState( + ResourceType.Document, + OperationType.Query, + queryRequestOptions, + spyClient + ); + spyClient.queryDocuments(getCollectionLink(isNameBased), query, dummyState, Document.class).blockFirst(); + assertThat(getSessionTokensInRequests().size()).isGreaterThanOrEqualTo(1); + assertThat(getSessionTokensInRequests().get(0)).isNotEmpty(); + assertThat(getSessionTokensInRequests().get(0)).doesNotContain(","); // making sure we have only one scope session token - // Session token validation for readAll with cross partition - spyClient.clearCapturedRequests(); - queryRequestOptions = new CosmosQueryRequestOptions(); + // Session token validation for readAll with partition query + spyClient.clearCapturedRequests(); + queryRequestOptions = new CosmosQueryRequestOptions(); + safeClose(dummyState); + dummyState = TestUtils.createDummyQueryFeedOperationState( + ResourceType.Document, + OperationType.ReadFeed, + queryRequestOptions, + spyClient + ); + spyClient.readAllDocuments( + getCollectionLink(isNameBased), + new PartitionKey(documentCreated.getId()), + dummyState, + Document.class).blockFirst(); + assertThat(getSessionTokensInRequests().size()).isEqualTo(1); + assertThat(getSessionTokensInRequests().get(0)).isNotEmpty(); + assertThat(getSessionTokensInRequests().get(0)).doesNotContain(","); // making sure we have only one scope session token - dummyState = TestUtils.createDummyQueryFeedOperationState( - ResourceType.Document, - OperationType.ReadFeed, - queryRequestOptions, - spyClient - ); + // Session token validation for readAll with cross partition + spyClient.clearCapturedRequests(); + queryRequestOptions = new CosmosQueryRequestOptions(); - spyClient.readDocuments(getCollectionLink(isNameBased), dummyState, Document.class).blockFirst(); - assertThat(getSessionTokensInRequests().size()).isGreaterThanOrEqualTo(1); - assertThat(getSessionTokensInRequests().get(0)).isNotEmpty(); - assertThat(getSessionTokensInRequests().get(0)).doesNotContain(","); // making sure we have only one scope session token + safeClose(dummyState); + dummyState = TestUtils.createDummyQueryFeedOperationState( + ResourceType.Document, + OperationType.ReadFeed, + queryRequestOptions, + spyClient + ); - // Session token validation for readMany with cross partition - spyClient.clearCapturedRequests(); - queryRequestOptions = new CosmosQueryRequestOptions(); - CosmosItemIdentity cosmosItemIdentity = new CosmosItemIdentity(new PartitionKey(documentCreated.getId()), documentCreated.getId()); - List cosmosItemIdentities = new ArrayList<>(); - cosmosItemIdentities.add(cosmosItemIdentity); - spyClient.readMany( - cosmosItemIdentities, - getCollectionLink(isNameBased), - TestUtils.createDummyQueryFeedOperationState(ResourceType.Document, OperationType.Query, queryRequestOptions, spyClient), - InternalObjectNode.class).block(); - assertThat(getSessionTokensInRequests().size()).isEqualTo(1); - assertThat(getSessionTokensInRequests().get(0)).isNotEmpty(); - assertThat(getSessionTokensInRequests().get(0)).doesNotContain(","); // making sure we have only one scope session token - // session token + spyClient.readDocuments(getCollectionLink(isNameBased), dummyState, Document.class).blockFirst(); + assertThat(getSessionTokensInRequests().size()).isGreaterThanOrEqualTo(1); + assertThat(getSessionTokensInRequests().get(0)).isNotEmpty(); + assertThat(getSessionTokensInRequests().get(0)).doesNotContain(","); // making sure we have only one scope session token - // Session token validation for create in Batch - if(isNameBased) { // Batch only work with name based url + // Session token validation for readMany with cross partition spyClient.clearCapturedRequests(); - Document document = newDocument(); - document.set("mypk", document.getId()); - ItemBatchOperation itemBatchOperation = new ItemBatchOperation(CosmosItemOperationType.CREATE, - documentCreated.getId(), new PartitionKey(documentCreated.getId()), new RequestOptions(), document); - List> itemBatchOperations = new ArrayList<>(); - itemBatchOperations.add(itemBatchOperation); - - Method method = SinglePartitionKeyServerBatchRequest.class.getDeclaredMethod("createBatchRequest", - PartitionKey.class, - List.class); - method.setAccessible(true); - SinglePartitionKeyServerBatchRequest serverBatchRequest = - (SinglePartitionKeyServerBatchRequest) method.invoke(SinglePartitionKeyServerBatchRequest.class, new PartitionKey(document.getId()), - itemBatchOperations); - spyClient - .executeBatchRequest( - getCollectionLink(isNameBased), - serverBatchRequest, - new RequestOptions(), - false, - true) - .block(); + queryRequestOptions = new CosmosQueryRequestOptions(); + CosmosItemIdentity cosmosItemIdentity = new CosmosItemIdentity(new PartitionKey(documentCreated.getId()), documentCreated.getId()); + List cosmosItemIdentities = new ArrayList<>(); + cosmosItemIdentities.add(cosmosItemIdentity); + safeClose(dummyState); + dummyState = TestUtils + .createDummyQueryFeedOperationState(ResourceType.Document, OperationType.Query, queryRequestOptions, spyClient) + spyClient.readMany( + cosmosItemIdentities, + getCollectionLink(isNameBased), + dummyState, + InternalObjectNode.class).block(); assertThat(getSessionTokensInRequests().size()).isEqualTo(1); assertThat(getSessionTokensInRequests().get(0)).isNotEmpty(); assertThat(getSessionTokensInRequests().get(0)).doesNotContain(","); // making sure we have only one scope session token + // session token + + // Session token validation for create in Batch + if (isNameBased) { // Batch only work with name based url + spyClient.clearCapturedRequests(); + Document document = newDocument(); + document.set("mypk", document.getId()); + ItemBatchOperation itemBatchOperation = new ItemBatchOperation(CosmosItemOperationType.CREATE, + documentCreated.getId(), new PartitionKey(documentCreated.getId()), new RequestOptions(), document); + List> itemBatchOperations = new ArrayList<>(); + itemBatchOperations.add(itemBatchOperation); + + Method method = SinglePartitionKeyServerBatchRequest.class.getDeclaredMethod("createBatchRequest", + PartitionKey.class, + List.class); + method.setAccessible(true); + SinglePartitionKeyServerBatchRequest serverBatchRequest = + (SinglePartitionKeyServerBatchRequest) method.invoke(SinglePartitionKeyServerBatchRequest.class, new PartitionKey(document.getId()), + itemBatchOperations); + spyClient + .executeBatchRequest( + getCollectionLink(isNameBased), + serverBatchRequest, + new RequestOptions(), + false, + true) + .block(); + assertThat(getSessionTokensInRequests().size()).isEqualTo(1); + assertThat(getSessionTokensInRequests().get(0)).isNotEmpty(); + assertThat(getSessionTokensInRequests().get(0)).doesNotContain(","); // making sure we have only one scope session token + } + } finally { + safeClose(dummyState); } } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestSuiteBase.java index 3da097f3213c..2208f14209aa 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestSuiteBase.java @@ -126,7 +126,9 @@ public Flux> queryDatabases(SqlQuerySpec query) { OperationType.Query, new CosmosQueryRequestOptions(), client); - return client.queryDatabases(query, state); + return client + .queryDatabases(query, state) + .doFinally(signal -> safeClose(state)); } @Override @@ -554,11 +556,15 @@ public static void deleteCollectionIfExists(AsyncDocumentClient client, String d new CosmosQueryRequestOptions(), client ); - List res = client.queryCollections("dbs/" + databaseId, - String.format("SELECT * FROM root r where r.id = '%s'", collectionId), state).single().block() - .getResults(); - if (!res.isEmpty()) { - deleteCollection(client, TestUtils.getCollectionNameLink(databaseId, collectionId)); + try { + List res = client.queryCollections("dbs/" + databaseId, + String.format("SELECT * FROM root r where r.id = '%s'", collectionId), state).single().block() + .getResults(); + if (!res.isEmpty()) { + deleteCollection(client, TestUtils.getCollectionNameLink(databaseId, collectionId)); + } + } finally { + safeClose(state); } } @@ -576,15 +582,19 @@ public static void deleteDocumentIfExists(AsyncDocumentClient client, String dat new CosmosQueryRequestOptions(), client ); - List res = client + try { + List res = client .queryDocuments( TestUtils.getCollectionNameLink(databaseId, collectionId), String.format("SELECT * FROM root r where r.id = '%s'", docId), state, Document.class) .single().block().getResults(); - if (!res.isEmpty()) { - deleteDocument(client, TestUtils.getDocumentNameLink(databaseId, collectionId, docId), pk, TestUtils.getCollectionNameLink(databaseId, collectionId)); + if (!res.isEmpty()) { + deleteDocument(client, TestUtils.getDocumentNameLink(databaseId, collectionId, docId), pk, TestUtils.getCollectionNameLink(databaseId, collectionId)); + } + } finally { + safeClose(state); } } @@ -601,11 +611,16 @@ public static void deleteUserIfExists(AsyncDocumentClient client, String databas new CosmosQueryRequestOptions(), client ); - List res = client + + try { + List res = client .queryUsers("dbs/" + databaseId, String.format("SELECT * FROM root r where r.id = '%s'", userId), state) .single().block().getResults(); - if (!res.isEmpty()) { - deleteUser(client, TestUtils.getUserNameLink(databaseId, userId)); + if (!res.isEmpty()) { + deleteUser(client, TestUtils.getUserNameLink(databaseId, userId)); + } + } finally { + safeClose(state); } } @@ -640,7 +655,8 @@ static protected Database createDatabaseIfNotExists(AsyncDocumentClient client, new CosmosQueryRequestOptions(), client ); - return client.queryDatabases(String.format("SELECT * FROM r where r.id = '%s'", databaseId), state).flatMap(p -> Flux.fromIterable(p.getResults())).switchIfEmpty( + try { + return client.queryDatabases(String.format("SELECT * FROM r where r.id = '%s'", databaseId), state).flatMap(p -> Flux.fromIterable(p.getResults())).switchIfEmpty( Flux.defer(() -> { Database databaseDefinition = new Database(); @@ -648,7 +664,10 @@ static protected Database createDatabaseIfNotExists(AsyncDocumentClient client, return client.createDatabase(databaseDefinition, null).map(ResourceResponse::getResource); }) - ).single().block(); + ).single().block(); + } finally { + safeClose(state); + } } static protected void safeDeleteDatabase(AsyncDocumentClient client, Database database) { @@ -674,14 +693,19 @@ static protected void safeDeleteAllCollections(AsyncDocumentClient client, Datab new CosmosQueryRequestOptions(), client ); - List collections = client.readCollections(database.getSelfLink(), state) - .flatMap(p -> Flux.fromIterable(p.getResults())) - .collectList() - .single() - .block(); - - for (DocumentCollection collection : collections) { - client.deleteCollection(collection.getSelfLink(), null).block().getResource(); + + try { + List collections = client.readCollections(database.getSelfLink(), state) + .flatMap(p -> Flux.fromIterable(p.getResults())) + .collectList() + .single() + .block(); + + for (DocumentCollection collection : collections) { + client.deleteCollection(collection.getSelfLink(), null).block().getResource(); + } + } finally { + safeClose(state); } } } @@ -716,6 +740,22 @@ static protected void safeCloseAsync(AsyncDocumentClient client) { } } + static protected void safeClose(QueryFeedOperationState state) { + if (state != null) { + safeClose(state.getClient()); + } + } + + static protected void safeClose(CosmosAsyncClient client) { + if (client != null) { + try { + client.close(); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + static protected void safeClose(AsyncDocumentClient client) { if (client != null) { try { diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/DCDocumentCrudTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/DCDocumentCrudTest.java index 77d68acb18bd..27969a39fd7c 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/DCDocumentCrudTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/DCDocumentCrudTest.java @@ -5,6 +5,7 @@ import com.azure.cosmos.ConsistencyLevel; import com.azure.cosmos.DirectConnectionConfig; import com.azure.cosmos.implementation.AsyncDocumentClient.Builder; +import com.azure.cosmos.implementation.QueryFeedOperationState; import com.azure.cosmos.implementation.TestUtils; import com.azure.cosmos.implementation.clienttelemetry.ClientTelemetry; import com.azure.cosmos.models.CosmosClientTelemetryConfig; @@ -228,20 +229,27 @@ public void crossPartitionQuery() { options.setMaxDegreeOfParallelism(-1); ModelBridgeInternal.setQueryRequestOptionsMaxItemCount(options, 100); - Flux> results = client.queryDocuments( - getCollectionLink(), - "SELECT * FROM r", - TestUtils.createDummyQueryFeedOperationState(ResourceType.Document, OperationType.Query, options, client), - Document.class); + QueryFeedOperationState dummyState = TestUtils + .createDummyQueryFeedOperationState(ResourceType.Document, OperationType.Query, options, client); - FeedResponseListValidator validator = new FeedResponseListValidator.Builder() + try { + Flux> results = client.queryDocuments( + getCollectionLink(), + "SELECT * FROM r", + dummyState, + Document.class); + + FeedResponseListValidator validator = new FeedResponseListValidator.Builder() .totalSize(documentList.size()) .exactlyContainsInAnyOrder(documentList.stream().map(Document::getResourceId).collect(Collectors.toList())).build(); - validateQuerySuccess(results, validator, QUERY_TIMEOUT); - validateNoDocumentQueryOperationThroughGateway(); - // validates only the first query for fetching query plan goes to gateway. - assertThat(client.getCapturedRequests().stream().filter(r -> r.getResourceType() == ResourceType.Document)).hasSize(1); + validateQuerySuccess(results, validator, QUERY_TIMEOUT); + validateNoDocumentQueryOperationThroughGateway(); + // validates only the first query for fetching query plan goes to gateway. + assertThat(client.getCapturedRequests().stream().filter(r -> r.getResourceType() == ResourceType.Document)).hasSize(1); + } finally { + safeClose(dummyState); + } } private void validateNoStoredProcExecutionOperationThroughGateway() { diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/BackPressureTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/BackPressureTest.java index abee1236ba68..f44802a44e4a 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/BackPressureTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/BackPressureTest.java @@ -269,25 +269,29 @@ public void before_BackPressureTest() throws Exception { rxClient ); - // increase throughput to max for a single partition collection to avoid throttling - // for bulk insert and later queries. - Offer offer = rxClient.queryOffers( + try { + // increase throughput to max for a single partition collection to avoid throttling + // for bulk insert and later queries. + Offer offer = rxClient.queryOffers( String.format("SELECT * FROM r WHERE r.offerResourceId = '%s'", createdCollection.read().block().getProperties().getResourceId()) - , state).take(1).map(FeedResponse::getResults).single().block().get(0); - offer.setThroughput(6000); - offer = rxClient.replaceOffer(offer).block().getResource(); - assertThat(offer.getThroughput()).isEqualTo(6000); - - ArrayList docDefList = new ArrayList<>(); - for(int i = 0; i < 1000; i++) { - docDefList.add(getDocumentDefinition(i)); - } + , state).take(1).map(FeedResponse::getResults).single().block().get(0); + offer.setThroughput(6000); + offer = rxClient.replaceOffer(offer).block().getResource(); + assertThat(offer.getThroughput()).isEqualTo(6000); + + ArrayList docDefList = new ArrayList<>(); + for (int i = 0; i < 1000; i++) { + docDefList.add(getDocumentDefinition(i)); + } - createdDocuments = bulkInsertBlocking(createdCollection, docDefList); + createdDocuments = bulkInsertBlocking(createdCollection, docDefList); - waitIfNeededForReplicasToCatchUp(getClientBuilder()); - warmUp(); + waitIfNeededForReplicasToCatchUp(getClientBuilder()); + warmUp(); + } finally { + safeClose(state); + } } private void warmUp() { diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OfferQueryTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OfferQueryTest.java index d0e79d6d3f28..20318c661f6c 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OfferQueryTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OfferQueryTest.java @@ -61,30 +61,36 @@ public void queryOffersWithFilter() throws Exception { String query = String.format("SELECT * from c where c.offerResourceId = '%s'", collectionResourceId); CosmosQueryRequestOptions options = new CosmosQueryRequestOptions(); - ModelBridgeInternal.setQueryRequestOptionsMaxItemCount(options, 2); - Flux> queryObservable = client.queryOffers( - query, - TestUtils.createDummyQueryFeedOperationState(ResourceType.Offer, OperationType.Query, options, client)); + QueryFeedOperationState dummyState = TestUtils + .createDummyQueryFeedOperationState(ResourceType.Offer, OperationType.Query, options, client); + try { + ModelBridgeInternal.setQueryRequestOptionsMaxItemCount(options, 2); + Flux> queryObservable = client.queryOffers( + query, + dummyState); - List allOffers = client - .readOffers(TestUtils.createDummyQueryFeedOperationState(ResourceType.Offer, OperationType.ReadFeed, options, client)) - .flatMap(f -> Flux.fromIterable(f.getResults())).collectList().single().block(); - List expectedOffers = allOffers.stream().filter(o -> collectionResourceId.equals(o.getString("offerResourceId"))).collect(Collectors.toList()); + List allOffers = client + .readOffers(TestUtils.createDummyQueryFeedOperationState(ResourceType.Offer, OperationType.ReadFeed, options, client)) + .flatMap(f -> Flux.fromIterable(f.getResults())).collectList().single().block(); + List expectedOffers = allOffers.stream().filter(o -> collectionResourceId.equals(o.getString("offerResourceId"))).collect(Collectors.toList()); - assertThat(expectedOffers).isNotEmpty(); + assertThat(expectedOffers).isNotEmpty(); - Integer maxItemCount = ModelBridgeInternal.getMaxItemCountFromQueryRequestOptions(options); - int expectedPageSize = (expectedOffers.size() + maxItemCount - 1) / maxItemCount; + Integer maxItemCount = ModelBridgeInternal.getMaxItemCountFromQueryRequestOptions(options); + int expectedPageSize = (expectedOffers.size() + maxItemCount - 1) / maxItemCount; - FeedResponseListValidator validator = new FeedResponseListValidator.Builder() + FeedResponseListValidator validator = new FeedResponseListValidator.Builder() .totalSize(expectedOffers.size()) .exactlyContainsInAnyOrder(expectedOffers.stream().map(d -> d.getResourceId()).collect(Collectors.toList())) .numberOfPages(expectedPageSize) .pageSatisfy(0, new FeedResponseValidator.Builder() - .requestChargeGreaterThanOrEqualTo(1.0).build()) + .requestChargeGreaterThanOrEqualTo(1.0).build()) .build(); - validateQuerySuccess(queryObservable, validator, 10000); + validateQuerySuccess(queryObservable, validator, 10000); + } finally { + safeClose(dummyState); + } } @Test(groups = { "query" }, timeOut = TIMEOUT * 10) @@ -97,11 +103,14 @@ public void queryOffersFilterMorePages() throws Exception { CosmosQueryRequestOptions options = new CosmosQueryRequestOptions(); ModelBridgeInternal.setQueryRequestOptionsMaxItemCount(options, 1); - Flux> queryObservable = client.queryOffers( - query, - TestUtils.createDummyQueryFeedOperationState(ResourceType.Offer, OperationType.Query, options, client)); + QueryFeedOperationState dummyState = TestUtils + .createDummyQueryFeedOperationState(ResourceType.Offer, OperationType.Query, options, client); + try { + Flux> queryObservable = client.queryOffers( + query, + dummyState); - List expectedOffers = client + List expectedOffers = client .readOffers(TestUtils.createDummyQueryFeedOperationState(ResourceType.Offer, OperationType.ReadFeed, new CosmosQueryRequestOptions(), client)) .flatMap(f -> Flux.fromIterable(f.getResults())) .collectList() @@ -109,20 +118,23 @@ public void queryOffersFilterMorePages() throws Exception { .stream().filter(o -> collectionResourceIds.contains(o.getOfferResourceId())) .collect(Collectors.toList()); - assertThat(expectedOffers).hasSize(createdCollections.size()); + assertThat(expectedOffers).hasSize(createdCollections.size()); - Integer maxItemCount = ModelBridgeInternal.getMaxItemCountFromQueryRequestOptions(options); - int expectedPageSize = (expectedOffers.size() + maxItemCount- 1) / maxItemCount; + Integer maxItemCount = ModelBridgeInternal.getMaxItemCountFromQueryRequestOptions(options); + int expectedPageSize = (expectedOffers.size() + maxItemCount - 1) / maxItemCount; - FeedResponseListValidator validator = new FeedResponseListValidator.Builder() + FeedResponseListValidator validator = new FeedResponseListValidator.Builder() .totalSize(expectedOffers.size()) .exactlyContainsInAnyOrder(expectedOffers.stream().map(d -> d.getResourceId()).collect(Collectors.toList())) .numberOfPages(expectedPageSize) .pageSatisfy(0, new FeedResponseValidator.Builder() - .requestChargeGreaterThanOrEqualTo(1.0).build()) + .requestChargeGreaterThanOrEqualTo(1.0).build()) .build(); - validateQuerySuccess(queryObservable, validator, 10000); + validateQuerySuccess(queryObservable, validator, 10000); + } finally { + safeClose(dummyState); + } } @Test(groups = { "query" }, timeOut = TIMEOUT) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OfferReadReplaceTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OfferReadReplaceTest.java index 6d6064d48b0a..c3ac8eb1e5a4 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OfferReadReplaceTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OfferReadReplaceTest.java @@ -3,6 +3,7 @@ package com.azure.cosmos.rx; import com.azure.cosmos.implementation.OperationType; +import com.azure.cosmos.implementation.QueryFeedOperationState; import com.azure.cosmos.implementation.ResourceType; import com.azure.cosmos.implementation.TestUtils; import com.azure.cosmos.models.CosmosQueryRequestOptions; @@ -42,48 +43,52 @@ public OfferReadReplaceTest(AsyncDocumentClient.Builder clientBuilder) { @Test(groups = { "emulator" }, timeOut = TIMEOUT) public void readAndReplaceOffer() { - List offers = client - .readOffers( - TestUtils.createDummyQueryFeedOperationState( - ResourceType.Offer, - OperationType.ReadFeed, - new CosmosQueryRequestOptions(), - client)) - .map(FeedResponse::getResults) - .flatMap(list -> Flux.fromIterable(list)).collectList().block(); - - int i; - for (i = 0; i < offers.size(); i++) { - if (offers.get(i).getOfferResourceId().equals(createdCollection.getResourceId())) { - break; + QueryFeedOperationState dummyState = TestUtils.createDummyQueryFeedOperationState( + ResourceType.Offer, + OperationType.ReadFeed, + new CosmosQueryRequestOptions(), + client); + try { + List offers = client + .readOffers(dummyState) + .map(FeedResponse::getResults) + .flatMap(list -> Flux.fromIterable(list)).collectList().block(); + + int i; + for (i = 0; i < offers.size(); i++) { + if (offers.get(i).getOfferResourceId().equals(createdCollection.getResourceId())) { + break; + } } - } - Offer rOffer = client.readOffer(offers.get(i).getSelfLink()).single().block().getResource(); - int oldThroughput = rOffer.getThroughput(); + Offer rOffer = client.readOffer(offers.get(i).getSelfLink()).single().block().getResource(); + int oldThroughput = rOffer.getThroughput(); - Mono> readObservable = client.readOffer(offers.get(i).getSelfLink()); + Mono> readObservable = client.readOffer(offers.get(i).getSelfLink()); - // validate offer read - ResourceResponseValidator validatorForRead = new ResourceResponseValidator.Builder() - .withOfferThroughput(oldThroughput) - .notNullEtag() - .build(); + // validate offer read + ResourceResponseValidator validatorForRead = new ResourceResponseValidator.Builder() + .withOfferThroughput(oldThroughput) + .notNullEtag() + .build(); - validateSuccess(readObservable, validatorForRead); + validateSuccess(readObservable, validatorForRead); - // update offer - int newThroughput = oldThroughput + 100; - offers.get(i).setThroughput(newThroughput); - Mono> replaceObservable = client.replaceOffer(offers.get(i)); + // update offer + int newThroughput = oldThroughput + 100; + offers.get(i).setThroughput(newThroughput); + Mono> replaceObservable = client.replaceOffer(offers.get(i)); - // validate offer replace - ResourceResponseValidator validatorForReplace = new ResourceResponseValidator.Builder() - .withOfferThroughput(newThroughput) - .notNullEtag() - .build(); + // validate offer replace + ResourceResponseValidator validatorForReplace = new ResourceResponseValidator.Builder() + .withOfferThroughput(newThroughput) + .notNullEtag() + .build(); - validateSuccess(replaceObservable, validatorForReplace); + validateSuccess(replaceObservable, validatorForReplace); + } finally { + safeClose(dummyState); + } } @BeforeClass(groups = { "emulator" }, timeOut = SETUP_TIMEOUT) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedOffersTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedOffersTest.java index aff4b8d6d536..1c9627502970 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedOffersTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedOffersTest.java @@ -99,18 +99,19 @@ public void before_ReadFeedOffersTest() { createCollections(client); } - allOffers = client.readOffers( - TestUtils.createDummyQueryFeedOperationState( - ResourceType.Offer, - OperationType.ReadFeed, - new CosmosQueryRequestOptions(), - client - ) - ) + QueryFeedOperationState dummyState = TestUtils.createDummyQueryFeedOperationState( + ResourceType.Offer, + OperationType.ReadFeed, + new CosmosQueryRequestOptions(), + client + ); + + allOffers = client.readOffers(dummyState) .map(FeedResponse::getResults) .collectList() .map(list -> list.stream().flatMap(Collection::stream).collect(Collectors.toList())) .single() + .doFinally(signal -> safeClose(dummyState)) .block(); } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ResourceTokenTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ResourceTokenTest.java index a94dfea5b85c..d42367a5ef49 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ResourceTokenTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ResourceTokenTest.java @@ -6,6 +6,7 @@ import com.azure.cosmos.ConsistencyLevel; import com.azure.cosmos.implementation.AsyncDocumentClient; import com.azure.cosmos.implementation.OperationType; +import com.azure.cosmos.implementation.QueryFeedOperationState; import com.azure.cosmos.implementation.ResourceType; import com.azure.cosmos.implementation.Utils; import com.azure.cosmos.implementation.clienttelemetry.ClientTelemetry; @@ -463,6 +464,7 @@ public void readDocumentFromCollPermissionWithDiffPartitionKey_WithException() t public void queryItemFromResourceToken(DocumentCollection documentCollection, Permission permission, PartitionKey partitionKey) throws Exception { AsyncDocumentClient asyncClientResourceToken = null; + QueryFeedOperationState dummyState = null; try { List permissionFeed = new ArrayList<>(); permissionFeed.add(permission); @@ -479,11 +481,19 @@ public void queryItemFromResourceToken(DocumentCollection documentCollection, Pe CosmosQueryRequestOptions queryRequestOptions = new CosmosQueryRequestOptions(); queryRequestOptions.setPartitionKey(partitionKey); + + dummyState = TestUtils + .createDummyQueryFeedOperationState( + ResourceType.Document, + OperationType.Query, + queryRequestOptions, + asyncClientResourceToken); + Flux> queryObservable = asyncClientResourceToken.queryDocuments( documentCollection.getAltLink(), "select * from c", - TestUtils.createDummyQueryFeedOperationState(ResourceType.Document, OperationType.Query, queryRequestOptions, asyncClientResourceToken), + dummyState, Document.class); FeedResponseListValidator validator = new FeedResponseListValidator.Builder() @@ -493,6 +503,7 @@ public void queryItemFromResourceToken(DocumentCollection documentCollection, Pe validateQuerySuccess(queryObservable, validator); } finally { + safeClose(dummyState); safeClose(asyncClientResourceToken); } } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java index d17e872db25e..62582a4511a4 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java @@ -32,6 +32,7 @@ import com.azure.cosmos.implementation.ImplementationBridgeHelpers; import com.azure.cosmos.implementation.InternalObjectNode; import com.azure.cosmos.implementation.PathParser; +import com.azure.cosmos.implementation.QueryFeedOperationState; import com.azure.cosmos.implementation.Resource; import com.azure.cosmos.implementation.TestConfigurations; import com.azure.cosmos.implementation.Utils; @@ -936,6 +937,12 @@ static protected void safeCloseAsync(CosmosAsyncClient client) { } } + static protected void safeClose(QueryFeedOperationState state) { + if (state != null) { + safeClose(state.getClient()); + } + } + static protected void safeClose(CosmosAsyncClient client) { if (client != null) { try { diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/AsyncDocumentClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/AsyncDocumentClient.java index f19ccb503027..9baeb17dbde8 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/AsyncDocumentClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/AsyncDocumentClient.java @@ -329,7 +329,14 @@ public AsyncDocumentClient build() { operationPolicies, isPerPartitionAutomaticFailoverEnabled); - client.init(state, null); + try { + client.init(state, null); + } catch (Throwable t) { + client.close(); + // TODO - should we map this to a CosmosException? + throw t; + } + return client; } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/FeedOperationState.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/FeedOperationState.java index 03922e5b1cec..900d6b2d6768 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/FeedOperationState.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/FeedOperationState.java @@ -155,6 +155,10 @@ public String getSpanName() { return ctxAccessor.getSpanName(this.ctxHolder.get()); } + public CosmosAsyncClient getClient() { + return this.cosmosAsyncClient; + } + public CosmosDiagnosticsContext getDiagnosticsContextSnapshot() { return this.ctxHolder.get(); } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java index ea81761b0411..82d7aeba54c3 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java @@ -417,7 +417,14 @@ private Mono toDocumentServiceResponse(Mono leakDetectionDebuggingEnabled ? bodyByteBuf.retain().touch(this) : bodyByteBuf.retain()) - .publishOn(CosmosSchedulers.TRANSPORT_RESPONSE_BOUNDED_ELASTIC); + .publishOn(CosmosSchedulers.TRANSPORT_RESPONSE_BOUNDED_ELASTIC) + .doOnDiscard(ByteBuf.class, buf -> { + if (buf.refCnt() > 0) { + // there could be a race with the catch in the .map operator below + // so, use safeRelease + io.netty.util.ReferenceCountUtil.safeRelease(buf); + } + }); return contentObservable .map(content -> { @@ -425,38 +432,48 @@ private Mono toDocumentServiceResponse(Mono 0) { + // Unwrap failed before StoreResponse took ownership -> release our retain + // there could be a race with the doOnDiscard above - so, use safeRelease + io.netty.util.ReferenceCountUtil.safeRelease(content); + } + + throw t; + } }) .single(); From 3ad7a16b858fdbdaeb4384c6be6f7ad38dc7ea87 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Mon, 10 Nov 2025 13:34:50 +0000 Subject: [PATCH 20/65] Update test-resources.json --- .../test-resources/cosmos-spring/test-resources.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/spring/spring-cloud-azure-integration-tests/test-resources/cosmos-spring/test-resources.json b/sdk/spring/spring-cloud-azure-integration-tests/test-resources/cosmos-spring/test-resources.json index ae625011a033..d96fc0f599fe 100644 --- a/sdk/spring/spring-cloud-azure-integration-tests/test-resources/cosmos-spring/test-resources.json +++ b/sdk/spring/spring-cloud-azure-integration-tests/test-resources/cosmos-spring/test-resources.json @@ -44,7 +44,7 @@ "newAccountName": "[toLower(concat(parameters('baseName'), '2'))]", "resourceId": "[resourceId('Microsoft.DocumentDB/databaseAccounts', variables('accountName'))]", "newResourceId": "[resourceId('Microsoft.DocumentDB/databaseAccounts', variables('newAccountName'))]", - "location": "[resourceGroup().location]" + "location": "westcentralus" }, "resources": [ { From f8643a359dff0ebab4ff6484a405ceb8a875922e Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Mon, 10 Nov 2025 14:39:01 +0100 Subject: [PATCH 21/65] Update sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosDiagnosticsE2ETest.java Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../test/java/com/azure/cosmos/CosmosDiagnosticsE2ETest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosDiagnosticsE2ETest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosDiagnosticsE2ETest.java index 786cfb12d249..b6e2e6ecb9cf 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosDiagnosticsE2ETest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosDiagnosticsE2ETest.java @@ -26,8 +26,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import io.micrometer.core.instrument.MeterRegistry; -import io.netty.buffer.ByteBuf; -import io.netty.buffer.ByteBufAllocator; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; From b4c2a9124b7d1c8d25c29594b4273cfe2fbfe66b Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Mon, 10 Nov 2025 14:42:52 +0100 Subject: [PATCH 22/65] [WIP] Fix Netty buffer and RxDocumentClientImpl leaks (#47213) * Initial plan * Improve JavaDoc phrasing in RxDocumentClientImpl Co-authored-by: FabianMeiswinkel <19165014+FabianMeiswinkel@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: FabianMeiswinkel <19165014+FabianMeiswinkel@users.noreply.github.com> Co-authored-by: Fabian Meiswinkel --- .../com/azure/cosmos/implementation/RxDocumentClientImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java index 107f2e18911b..4b3ba08ef61a 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java @@ -1395,7 +1395,7 @@ private void removeFromActiveClients() { } /** - * Returns a snapshot of the active clients. The key is the clientId, the value the callstack shows from + * Returns a snapshot of the active clients. The key is the clientId, the value is the callstack showing * where the client was created. * @return a snapshot of the active clients. */ From 639b41eda0be0e8703df52d70b3100b4d119c89c Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Mon, 10 Nov 2025 13:52:30 +0000 Subject: [PATCH 23/65] NITs --- .../cosmos/encryption/CosmosEncryptionAsyncClientTest.java | 2 +- .../src/test/java/com/azure/cosmos/CosmosAsyncClientTest.java | 4 ++-- .../src/test/java/com/azure/cosmos/DocumentClientTest.java | 4 ++-- .../main/java/com/azure/cosmos/implementation/Configs.java | 3 ++- .../com/azure/cosmos/implementation/RxDocumentClientImpl.java | 2 +- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncClientTest.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncClientTest.java index 0c39b750776d..d396a824659b 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncClientTest.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncClientTest.java @@ -97,7 +97,7 @@ public void afterClassSetupLeakDetection() { } if (sb.length() > 0) { - String msg = "\"COSMOS CLIENT LEAKS detected in test class: " + String msg = "COSMOS CLIENT LEAKS detected in test class: " + this.getClass().getCanonicalName() + "\n\n" + sb; diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosAsyncClientTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosAsyncClientTest.java index 554b52f3804c..24fe8f9b06b6 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosAsyncClientTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosAsyncClientTest.java @@ -97,7 +97,7 @@ public void afterClassSetupLeakDetection() { } if (sb.length() > 0) { - String msg = "\"COSMOS CLIENT LEAKS detected in test class: " + String msg = "COSMOS CLIENT LEAKS detected in test class: " + this.getClass().getCanonicalName() + "\n\n" + sb; @@ -113,7 +113,7 @@ public void afterClassSetupLeakDetection() { sb.append(leak).append("\n"); } - String msg = "\"NETTY LEAKS detected in test class: " + String msg = "NETTY LEAKS detected in test class: " + this.getClass().getCanonicalName() + sb; diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/DocumentClientTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/DocumentClientTest.java index 1050c0412f94..000bfaffc0f0 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/DocumentClientTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/DocumentClientTest.java @@ -91,7 +91,7 @@ public void afterClassSetupLeakDetection() { } if (sb.length() > 0) { - String msg = "\"COSMOS CLIENT LEAKS detected in test class: " + String msg = "COSMOS CLIENT LEAKS detected in test class: " + this.getClass().getCanonicalName() + "\n\n" + sb; @@ -107,7 +107,7 @@ public void afterClassSetupLeakDetection() { sb.append(leak).append("\n"); } - String msg = "\"NETTY LEAKS detected in test class: " + String msg = "NETTY LEAKS detected in test class: " + this.getClass().getCanonicalName() + sb; diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java index 5de382166cf6..b656b84e66db 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java @@ -374,7 +374,8 @@ public class Configs { public static final String DEFAULT_OTEL_SPAN_ATTRIBUTE_NAMING_SCHEME = "All"; - private static final boolean DEFAULT_CLIENT_LEAK_DETECTION_ENABLED = true; // TODO @fabianm - revert before merging - just cheaper to enable than making test config changes + // TODO @fabianm - Make test changes to enable leak detection from CI pipeline tests + private static final boolean DEFAULT_CLIENT_LEAK_DETECTION_ENABLED = false; private static final String CLIENT_LEAK_DETECTION_ENABLED = "COSMOS.CLIENT_LEAK_DETECTION_ENABLED"; public static int getCPUCnt() { diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java index 4b3ba08ef61a..25f02086acc3 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java @@ -203,7 +203,7 @@ public class RxDocumentClientImpl implements AsyncDocumentClient, IAuthorization private static final Object staticLock = new Object(); - // A map containing the clientId with the callstack form where the Client was initialized. + // A map containing the clientId with the callstack from where the Client was initialized. // this can help to identify where clients leak. // The leak detection via System property "COSMOS.CLIENT_LEAK_DETECTION_ENABLED" is disabled by // default - CI pipeline tests will enable it. From 68bb25e8bbeee3d7ec0d2ffeefd809ef51246646 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Mon, 10 Nov 2025 15:01:41 +0100 Subject: [PATCH 24/65] Move static block to class level in cosmos-encryption TestSuiteBase (#47216) * Initial plan * Move CosmosNettyLeakDetectorFactory.ingestIntoNetty() to class-level static block Co-authored-by: FabianMeiswinkel <19165014+FabianMeiswinkel@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: FabianMeiswinkel <19165014+FabianMeiswinkel@users.noreply.github.com> --- .../java/com/azure/cosmos/encryption/TestSuiteBase.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java index 3457351177d4..d4756008f977 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java @@ -170,6 +170,7 @@ protected static CosmosAsyncContainer getSharedSinglePartitionCosmosContainer(Co } static { + CosmosNettyLeakDetectorFactory.ingestIntoNetty(); accountConsistency = parseConsistency(TestConfigurations.CONSISTENCY); desiredConsistencies = immutableListOrNull( ObjectUtils.defaultIfNull(parseDesiredConsistencies(TestConfigurations.DESIRED_CONSISTENCIES), @@ -1482,9 +1483,4 @@ protected static void validateResponse(EncryptionPojo originalItem, EncryptionPo assertThat(result.getSensitiveChildPojo2DArray()[0][0].getSensitiveStringArray()).isEqualTo(originalItem.getSensitiveChildPojo2DArray()[0][0].getSensitiveStringArray()); assertThat(result.getSensitiveChildPojo2DArray()[0][0].getSensitiveString3DArray()).isEqualTo(originalItem.getSensitiveChildPojo2DArray()[0][0].getSensitiveString3DArray()); } - - - static { - CosmosNettyLeakDetectorFactory.ingestIntoNetty(); - } } From 6f12a81578591bb3566883e10eeeac0229be8a2f Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Mon, 10 Nov 2025 21:30:27 +0000 Subject: [PATCH 25/65] Update SessionTest.java --- .../test/java/com/azure/cosmos/implementation/SessionTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/SessionTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/SessionTest.java index f1482ac5accf..eb1ff945e297 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/SessionTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/SessionTest.java @@ -286,7 +286,7 @@ public void partitionedSessionToken(boolean isNameBased) throws NoSuchMethodExce cosmosItemIdentities.add(cosmosItemIdentity); safeClose(dummyState); dummyState = TestUtils - .createDummyQueryFeedOperationState(ResourceType.Document, OperationType.Query, queryRequestOptions, spyClient) + .createDummyQueryFeedOperationState(ResourceType.Document, OperationType.Query, queryRequestOptions, spyClient); spyClient.readMany( cosmosItemIdentities, getCollectionLink(isNameBased), From 3caade1c291b1e467d57c76d28249386cfa85488 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Mon, 10 Nov 2025 22:58:40 +0000 Subject: [PATCH 26/65] Update tests.yml --- sdk/cosmos/tests.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sdk/cosmos/tests.yml b/sdk/cosmos/tests.yml index 17af3836ec2d..c52375d92d69 100644 --- a/sdk/cosmos/tests.yml +++ b/sdk/cosmos/tests.yml @@ -33,7 +33,7 @@ extends: TestResultsFiles: '**/junitreports/TEST-*.xml' AdditionalVariables: - name: AdditionalArgs - value: '-DCOSMOS.CLIENT_TELEMETRY_ENDPOINT=$(cosmos-client-telemetry-endpoint) -DCOSMOS.CLIENT_TELEMETRY_COSMOS_ACCOUNT=$(cosmos-client-telemetry-cosmos-account)' + value: '-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true' - template: /eng/pipelines/templates/stages/archetype-sdk-tests-isolated.yml parameters: @@ -65,7 +65,7 @@ extends: TestResultsFiles: '**/junitreports/TEST-*.xml' AdditionalVariables: - name: AdditionalArgs - value: '-DCOSMOS.CLIENT_TELEMETRY_ENDPOINT=$(cosmos-client-telemetry-endpoint) -DCOSMOS.CLIENT_TELEMETRY_COSMOS_ACCOUNT=$(cosmos-client-telemetry-cosmos-account) -DCOSMOS.HTTP2_ENABLED=true' + value: '-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -DCOSMOS.HTTP2_ENABLED=true' - template: /eng/pipelines/templates/stages/archetype-sdk-tests-isolated.yml parameters: @@ -97,7 +97,7 @@ extends: TestResultsFiles: '**/junitreports/TEST-*.xml' AdditionalVariables: - name: AdditionalArgs - value: '-DCOSMOS.CLIENT_TELEMETRY_ENDPOINT=$(cosmos-client-telemetry-endpoint) -DCOSMOS.CLIENT_TELEMETRY_COSMOS_ACCOUNT=$(cosmos-client-telemetry-cosmos-account) -DACCOUNT_HOST=$(thinclient-test-endpoint) -DACCOUNT_KEY=$(thinclient-test-key) -DCOSMOS.THINCLIENT_ENABLED=true -DCOSMOS.HTTP2_ENABLED=true' + value: '-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -DACCOUNT_HOST=$(thinclient-test-endpoint) -DACCOUNT_KEY=$(thinclient-test-key) -DCOSMOS.THINCLIENT_ENABLED=true -DCOSMOS.HTTP2_ENABLED=true' - template: /eng/pipelines/templates/stages/archetype-sdk-tests-isolated.yml parameters: @@ -129,7 +129,7 @@ extends: TestResultsFiles: '**/junitreports/TEST-*.xml' AdditionalVariables: - name: AdditionalArgs - value: '-DCOSMOS.CLIENT_TELEMETRY_ENDPOINT=$(cosmos-client-telemetry-endpoint) -DCOSMOS.CLIENT_TELEMETRY_COSMOS_ACCOUNT=$(cosmos-client-telemetry-cosmos-account) -DACCOUNT_HOST=$(thin-client-canary-multi-region-session-endpoint) -DACCOUNT_KEY=$(thin-client-canary-multi-region-session-key) -DCOSMOS.THINCLIENT_ENABLED=true' + value: '-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -DACCOUNT_HOST=$(thin-client-canary-multi-region-session-endpoint) -DACCOUNT_KEY=$(thin-client-canary-multi-region-session-key) -DCOSMOS.THINCLIENT_ENABLED=true' - template: /eng/pipelines/templates/stages/archetype-sdk-tests-isolated.yml parameters: @@ -161,7 +161,7 @@ extends: TestResultsFiles: '**/junitreports/TEST-*.xml' AdditionalVariables: - name: AdditionalArgs - value: '-DCOSMOS.CLIENT_TELEMETRY_ENDPOINT=$(cosmos-client-telemetry-endpoint) -DCOSMOS.CLIENT_TELEMETRY_COSMOS_ACCOUNT=$(cosmos-client-telemetry-cosmos-account) -DACCOUNT_HOST=$(thin-client-canary-multi-writer-session-endpoint) -DACCOUNT_KEY=$(thin-client-canary-multi-writer-session-key) -DCOSMOS.THINCLIENT_ENABLED=true' + value: '-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -DACCOUNT_HOST=$(thin-client-canary-multi-writer-session-endpoint) -DACCOUNT_KEY=$(thin-client-canary-multi-writer-session-key) -DCOSMOS.THINCLIENT_ENABLED=true' - template: /eng/pipelines/templates/stages/archetype-sdk-tests-isolated.yml parameters: @@ -186,7 +186,7 @@ extends: TestOptions: '$(ProfileFlag) -DskipCompile=true -DskipTestCompile=true -DcreateSourcesJar=false' AdditionalVariables: - name: AdditionalArgs - value: '-DCOSMOS.CLIENT_TELEMETRY_ENDPOINT=$(cosmos-client-telemetry-endpoint) -DCOSMOS.CLIENT_TELEMETRY_COSMOS_ACCOUNT=$(cosmos-client-telemetry-cosmos-account)' + value: '-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=false' - template: /eng/pipelines/templates/stages/archetype-sdk-tests-isolated.yml parameters: @@ -212,5 +212,5 @@ extends: TestOptions: '$(ProfileFlag) $(AdditionalArgs)' AdditionalVariables: - name: AdditionalArgs - value: '-DCOSMOS.CLIENT_TELEMETRY_ENDPOINT=$(cosmos-client-telemetry-endpoint) -DCOSMOS.CLIENT_TELEMETRY_COSMOS_ACCOUNT=$(cosmos-client-telemetry-cosmos-account)' + value: '-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true' From 8a798ff3cf91597c34502d2efeb25bd745ac05c9 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Tue, 11 Nov 2025 00:08:14 +0000 Subject: [PATCH 27/65] Update CosmosNettyLeakDetectorFactory.java --- .../cosmos/encryption/CosmosNettyLeakDetectorFactory.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java index c2b5f179131f..8fcd4f418548 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java @@ -2,6 +2,7 @@ // Licensed under the MIT License. package com.azure.cosmos.encryption; +import com.azure.cosmos.implementation.Configs; import com.azure.cosmos.implementation.StackTraceUtil; import io.netty.util.ResourceLeakDetector; import io.netty.util.ResourceLeakDetectorFactory; @@ -58,7 +59,9 @@ public static void ingestIntoNetty() { // install custom reporter ResourceLeakDetectorFactory.setResourceLeakDetectorFactory(new CosmosNettyLeakDetectorFactory()); - logger.info("NETTY LEAK detection initialized"); + logger.info( + "NETTY LEAK detection initialized, CosmosClient leak detection enabled: {}", + Configs.isClientLeakDetectionEnabled()); isInitialized = true; } } From 8059e0ef7531311085fe71036cfa7fd058f10805 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Tue, 11 Nov 2025 00:32:19 +0000 Subject: [PATCH 28/65] Test config --- sdk/cosmos/live-platform-matrix.json | 16 ++++++++-------- sdk/cosmos/tests.yml | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/sdk/cosmos/live-platform-matrix.json b/sdk/cosmos/live-platform-matrix.json index 7f3e3af80997..613390ddd671 100644 --- a/sdk/cosmos/live-platform-matrix.json +++ b/sdk/cosmos/live-platform-matrix.json @@ -14,7 +14,7 @@ "-Pcircuit-breaker-read-all-read-many": "CircuitBreakerReadAllAndReadMany", "-Pmulti-region": "MultiRegion", "-Plong": "Long", - "-DargLine=\"-Dazure.cosmos.directModeProtocol=Tcp\"": "TCP", + "-DargLine=\"-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -Dazure.cosmos.directModeProtocol=Tcp\"": "TCP", "Session": "", "ubuntu": "", "@{ enableMultipleWriteLocations = $false; defaultConsistencyLevel = 'Session' }": "", @@ -38,7 +38,7 @@ } }, "AdditionalArgs": [ - "-DargLine=\"-Dazure.cosmos.directModeProtocol=Tcp\"" + "-DargLine=\"-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -Dazure.cosmos.directModeProtocol=Tcp\"" ], "ProfileFlag": "-Pe2e", "Agent": { @@ -52,7 +52,7 @@ "ProfileFlag": [ "-Pcfp-split", "-Psplit", "-Pquery", "-Pfast", "-Pdirect" ], "ArmTemplateParameters": "@{ enableMultipleWriteLocations = $false; defaultConsistencyLevel = 'Session' }", "AdditionalArgs": [ - "-DargLine=\"-Dio.netty.handler.ssl.noOpenSsl=true\"" + "-DargLine=\"-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -Dio.netty.handler.ssl.noOpenSsl=true\"" ], "Agent": { "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" } @@ -71,7 +71,7 @@ { "DESIRED_CONSISTENCY": "BoundedStaleness", "ACCOUNT_CONSISTENCY": "Strong", - "AdditionalArgs": "-DargLine=\"-Dazure.cosmos.directModeProtocol=Tcp\"", + "AdditionalArgs": "-DargLine=\"-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -Dazure.cosmos.directModeProtocol=Tcp\"", "ProfileFlag": "-Pe2e", "ArmTemplateParameters": "@{ enableMultipleWriteLocations = $false; defaultConsistencyLevel = 'Strong' }", "Agent": { @@ -109,7 +109,7 @@ }, "PROTOCOLS": "[\"Tcp\"]", "ProfileFlag": [ "-Pmulti-master" ], - "AdditionalArgs": "\"-DCOSMOS.PARTITION_LEVEL_CIRCUIT_BREAKER_DEFAULT_CONFIG_OPT_IN=TRUE\"", + "AdditionalArgs": "\"-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -DCOSMOS.PARTITION_LEVEL_CIRCUIT_BREAKER_DEFAULT_CONFIG_OPT_IN=TRUE\"", "Agent": { "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" } } @@ -125,7 +125,7 @@ }, "PROTOCOLS": "[\"Tcp\"]", "ProfileFlag": [ "-Pfi-multi-master" ], - "AdditionalArgs": "\"-DCOSMOS.PARTITION_LEVEL_CIRCUIT_BREAKER_DEFAULT_CONFIG_OPT_IN=TRUE\"", + "AdditionalArgs": "\"-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -DCOSMOS.PARTITION_LEVEL_CIRCUIT_BREAKER_DEFAULT_CONFIG_OPT_IN=TRUE\"", "Agent": { "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" } } @@ -141,7 +141,7 @@ }, "PROTOCOLS": "[\"Tcp\"]", "ProfileFlag": [ "-Pmulti-master" ], - "AdditionalArgs": "\"-DCOSMOS.PARTITION_LEVEL_CIRCUIT_BREAKER_DEFAULT_CONFIG_OPT_IN=FALSE\"", + "AdditionalArgs": "\"-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -DCOSMOS.PARTITION_LEVEL_CIRCUIT_BREAKER_DEFAULT_CONFIG_OPT_IN=FALSE\"", "Agent": { "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" } } @@ -157,7 +157,7 @@ }, "PROTOCOLS": "[\"Tcp\"]", "ProfileFlag": [ "-Pfi-multi-master" ], - "AdditionalArgs": "\"-DCOSMOS.PARTITION_LEVEL_CIRCUIT_BREAKER_DEFAULT_CONFIG_OPT_IN=FALSE\"", + "AdditionalArgs": "\"-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -DCOSMOS.PARTITION_LEVEL_CIRCUIT_BREAKER_DEFAULT_CONFIG_OPT_IN=FALSE\"", "Agent": { "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" } } diff --git a/sdk/cosmos/tests.yml b/sdk/cosmos/tests.yml index c52375d92d69..69d782fcc9a0 100644 --- a/sdk/cosmos/tests.yml +++ b/sdk/cosmos/tests.yml @@ -183,7 +183,7 @@ extends: safeName: azurespringdatacosmos TimeoutInMinutes: 90 TestGoals: 'verify' - TestOptions: '$(ProfileFlag) -DskipCompile=true -DskipTestCompile=true -DcreateSourcesJar=false' + TestOptions: '$(ProfileFlag) $(AdditionalArgs) -DskipCompile=true -DskipTestCompile=true -DcreateSourcesJar=false' AdditionalVariables: - name: AdditionalArgs value: '-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=false' From 5ea5a927c0df385cdb628fa0642d915a8be67859 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Tue, 11 Nov 2025 00:36:35 +0000 Subject: [PATCH 29/65] Update CosmosNettyLeakDetectorFactory.java --- .../com/azure/cosmos/CosmosNettyLeakDetectorFactory.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java index c94a542edfff..1182209dc752 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java @@ -2,6 +2,7 @@ // Licensed under the MIT License. package com.azure.cosmos; +import com.azure.cosmos.implementation.Configs; import com.azure.cosmos.implementation.StackTraceUtil; import io.netty.util.ResourceLeakDetector; import io.netty.util.ResourceLeakDetectorFactory; @@ -58,7 +59,9 @@ public static void ingestIntoNetty() { // install custom reporter ResourceLeakDetectorFactory.setResourceLeakDetectorFactory(new CosmosNettyLeakDetectorFactory()); - logger.info("NETTY LEAK detection initialized"); + logger.info( + "NETTY LEAK detection initialized, CosmosClient leak detection enabled: {}", + Configs.isClientLeakDetectionEnabled()); isInitialized = true; } } From 82216f93d972a2ac71f6f9928456734b792637d2 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Tue, 11 Nov 2025 11:30:49 +0000 Subject: [PATCH 30/65] Updating TestNG --- eng/versioning/external_dependencies.txt | 2 +- sdk/cosmos/azure-cosmos-benchmark/pom.xml | 5 +- sdk/cosmos/azure-cosmos-encryption/README.md | 2 +- sdk/cosmos/azure-cosmos-encryption/pom.xml | 5 +- .../CosmosEncryptionAsyncClientTest.java | 76 +--------------- .../cosmos/encryption/TestSuiteBase.java | 10 +-- sdk/cosmos/azure-cosmos-kafka-connect/pom.xml | 5 +- .../pom.xml | 3 + .../azure-cosmos-spark_3-3_2-12/pom.xml | 3 + .../azure-cosmos-spark_3-4_2-12/pom.xml | 3 + .../azure-cosmos-spark_3-5_2-12/pom.xml | 3 + sdk/cosmos/azure-cosmos-spark_3/pom.xml | 3 + sdk/cosmos/azure-cosmos-test/pom.xml | 5 +- sdk/cosmos/azure-cosmos-tests/pom.xml | 5 +- .../azure/cosmos/CosmosAsyncClientTest.java | 79 +--------------- .../CosmosNettyLeakDetectorFactory.java | 90 +++++++++++++++++-- .../com/azure/cosmos/DocumentClientTest.java | 78 +--------------- .../cosmos/implementation/TestSuiteBase.java | 11 +-- .../com/azure/cosmos/rx/TestSuiteBase.java | 10 +-- sdk/cosmos/azure-cosmos/pom.xml | 2 +- sdk/cosmos/fabric-cosmos-spark-auth_3/pom.xml | 3 + 21 files changed, 133 insertions(+), 270 deletions(-) diff --git a/eng/versioning/external_dependencies.txt b/eng/versioning/external_dependencies.txt index 47e532c6ee5c..09453aa8bebb 100644 --- a/eng/versioning/external_dependencies.txt +++ b/eng/versioning/external_dependencies.txt @@ -152,7 +152,7 @@ org.junit.jupiter:junit-jupiter-params;5.13.4 org.junit.platform:junit-platform-launcher;1.13.4 org.openjdk.jmh:jmh-core;1.37 org.openjdk.jmh:jmh-generator-annprocess;1.37 -org.testng:testng;7.5.1 +org.testng:testng;7.11.0 uk.org.webcompere:system-stubs-jupiter;2.0.2 com.google.truth:truth;1.1.3 diff --git a/sdk/cosmos/azure-cosmos-benchmark/pom.xml b/sdk/cosmos/azure-cosmos-benchmark/pom.xml index 7ae0b50faafb..cd7b43b5e9d1 100644 --- a/sdk/cosmos/azure-cosmos-benchmark/pom.xml +++ b/sdk/cosmos/azure-cosmos-benchmark/pom.xml @@ -165,7 +165,7 @@ Licensed under the MIT License. org.testng testng - 7.5.1 + 7.11.0 test @@ -213,6 +213,9 @@ Licensed under the MIT License. true + + true + diff --git a/sdk/cosmos/azure-cosmos-encryption/README.md b/sdk/cosmos/azure-cosmos-encryption/README.md index 97bc4c8ef534..d460dca455f1 100644 --- a/sdk/cosmos/azure-cosmos-encryption/README.md +++ b/sdk/cosmos/azure-cosmos-encryption/README.md @@ -12,7 +12,7 @@ The Azure Cosmos Encryption Plugin is used for encrypting data with a user-provi com.azure azure-cosmos-encryption - 2.24.0 + 2.25.0-beta.1 ``` [//]: # ({x-version-update-end}) diff --git a/sdk/cosmos/azure-cosmos-encryption/pom.xml b/sdk/cosmos/azure-cosmos-encryption/pom.xml index a9984111c017..584758735f6a 100644 --- a/sdk/cosmos/azure-cosmos-encryption/pom.xml +++ b/sdk/cosmos/azure-cosmos-encryption/pom.xml @@ -130,7 +130,7 @@ Licensed under the MIT License. org.testng testng - 7.5.1 + 7.11.0 test @@ -227,6 +227,9 @@ Licensed under the MIT License. true + + true + diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncClientTest.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncClientTest.java index d396a824659b..d91fdd7c7af8 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncClientTest.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosEncryptionAsyncClientTest.java @@ -20,6 +20,7 @@ import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Listeners; import java.lang.management.BufferPoolMXBean; import java.lang.management.ManagementFactory; @@ -29,16 +30,15 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; +@Listeners({TestNGLogListener.class, CosmosNettyLeakDetectorFactory.class}) public abstract class CosmosEncryptionAsyncClientTest implements ITest { protected static Logger logger = LoggerFactory.getLogger(CosmosEncryptionAsyncClientTest.class.getSimpleName()); protected static final int SUITE_SETUP_TIMEOUT = 120000; private static final ImplementationBridgeHelpers.CosmosClientBuilderHelper.CosmosClientBuilderAccessor cosmosClientBuilderAccessor = ImplementationBridgeHelpers.CosmosClientBuilderHelper.getCosmosClientBuilderAccessor(); - private final static AtomicInteger instancesUsed = new AtomicInteger(0); private final CosmosClientBuilder clientBuilder; private String testName; - private volatile Map activeClientsAtBegin = new HashMap<>(); public CosmosEncryptionAsyncClientTest() { this(new CosmosClientBuilder()); @@ -52,78 +52,6 @@ public final CosmosClientBuilder getClientBuilder() { return this.clientBuilder; } - @BeforeClass(groups = {"fast", "long", "direct", "multi-master", "encryption"}, timeOut = SUITE_SETUP_TIMEOUT) - - public void beforeClassSetupLeakDetection() { - if (instancesUsed.getAndIncrement() == 0) { - this.activeClientsAtBegin = RxDocumentClientImpl.getActiveClientsSnapshot(); - this.logMemoryUsage("BEFORE"); - } - } - - private void logMemoryUsage(String name) { - long pooledDirectBytes = PooledByteBufAllocator.DEFAULT.metric() - .directArenas().stream() - .mapToLong(io.netty.buffer.PoolArenaMetric::numActiveBytes) - .sum(); - - long used = PlatformDependent.usedDirectMemory(); - long max = PlatformDependent.maxDirectMemory(); - logger.info("MEMORY USAGE: {}:{}", this.getClass().getCanonicalName(), name); - logger.info("Netty Direct Memory: {}/{}/{} bytes", used, pooledDirectBytes, max); - for (BufferPoolMXBean pool : ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class)) { - logger.info("Pool {}: used={} bytes, capacity={} bytes, count={}", - pool.getName(), pool.getMemoryUsed(), pool.getTotalCapacity(), pool.getCount()); - } - } - - @AfterClass(groups = {"fast", "long", "direct", "multi-master", "encryption"}, timeOut = SUITE_SETUP_TIMEOUT) - public void afterClassSetupLeakDetection() { - if (instancesUsed.decrementAndGet() == 0) { - Map leakedClientSnapshotNow = RxDocumentClientImpl.getActiveClientsSnapshot(); - StringBuilder sb = new StringBuilder(); - Map leakedClientSnapshotAtBegin = activeClientsAtBegin; - - for (Integer clientId : leakedClientSnapshotNow.keySet()) { - if (!leakedClientSnapshotAtBegin.containsKey(clientId)) { - // this client was leaked in this class - sb - .append("CosmosClient [") - .append(clientId) - .append("] leaked. Callstack of initialization:\n") - .append(leakedClientSnapshotNow.get(clientId)) - .append("\n\n"); - } - } - - if (sb.length() > 0) { - String msg = "COSMOS CLIENT LEAKS detected in test class: " - + this.getClass().getCanonicalName() - + "\n\n" - + sb; - - logger.error(msg); - // fail(msg); - } - - List nettyLeaks = CosmosNettyLeakDetectorFactory.resetIdentifiedLeaks(); - if (nettyLeaks.size() > 0) { - sb.append("\n"); - for (String leak : nettyLeaks) { - sb.append(leak).append("\n"); - } - - String msg = "NETTY LEAKS detected in test class: " - + this.getClass().getCanonicalName() - + sb; - - logger.error(msg); - // fail(msg); - } - this.logMemoryUsage("AFTER"); - } - } - @Override public final String getTestName() { return this.testName; diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java index d4756008f977..aae4f2b707c2 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java @@ -82,8 +82,7 @@ import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.spy; -@Listeners({TestNGLogListener.class}) -public class TestSuiteBase extends CosmosEncryptionAsyncClientTest { +public abstract class TestSuiteBase extends CosmosEncryptionAsyncClientTest { private static final int DEFAULT_BULK_INSERT_CONCURRENCY_LEVEL = 500; private static final ObjectMapper objectMapper = new ObjectMapper(); @@ -257,13 +256,6 @@ public void beforeSuite() { } } - @BeforeSuite(groups = {"unit"}) - public void parallelizeUnitTests(ITestContext context) { - // TODO: Parallelization was disabled due to flaky tests. Re-enable after fixing the flaky tests. -// context.getSuite().getXmlSuite().setParallel(XmlSuite.ParallelMode.CLASSES); -// context.getSuite().getXmlSuite().setThreadCount(Runtime.getRuntime().availableProcessors()); - } - @AfterSuite(groups = {"fast", "long", "direct", "multi-master", "encryption"}, timeOut = SUITE_SHUTDOWN_TIMEOUT) public void afterSuite() { diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml b/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml index 1693af8dc8fb..6e3dfe5465ec 100644 --- a/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml +++ b/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml @@ -160,7 +160,7 @@ Licensed under the MIT License. org.testng testng - 7.5.1 + 7.11.0 test @@ -286,6 +286,9 @@ Licensed under the MIT License. true + + true + diff --git a/sdk/cosmos/azure-cosmos-spark-account-data-resolver-sample/pom.xml b/sdk/cosmos/azure-cosmos-spark-account-data-resolver-sample/pom.xml index 7391f7308117..9951686b5b27 100644 --- a/sdk/cosmos/azure-cosmos-spark-account-data-resolver-sample/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark-account-data-resolver-sample/pom.xml @@ -703,6 +703,9 @@ **/*Spec.* true + + true + diff --git a/sdk/cosmos/azure-cosmos-spark_3-3_2-12/pom.xml b/sdk/cosmos/azure-cosmos-spark_3-3_2-12/pom.xml index 948990bafe7f..7a7d299de19f 100644 --- a/sdk/cosmos/azure-cosmos-spark_3-3_2-12/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark_3-3_2-12/pom.xml @@ -125,6 +125,9 @@ **/*Spec.* true + + true + diff --git a/sdk/cosmos/azure-cosmos-spark_3-4_2-12/pom.xml b/sdk/cosmos/azure-cosmos-spark_3-4_2-12/pom.xml index fcecfd210ea5..98483ad3ad1b 100644 --- a/sdk/cosmos/azure-cosmos-spark_3-4_2-12/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark_3-4_2-12/pom.xml @@ -125,6 +125,9 @@ **/*Spec.* true + + true + diff --git a/sdk/cosmos/azure-cosmos-spark_3-5_2-12/pom.xml b/sdk/cosmos/azure-cosmos-spark_3-5_2-12/pom.xml index 435e9fd672e5..b2eff8159e95 100644 --- a/sdk/cosmos/azure-cosmos-spark_3-5_2-12/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark_3-5_2-12/pom.xml @@ -125,6 +125,9 @@ **/*Spec.* true + + true + diff --git a/sdk/cosmos/azure-cosmos-spark_3/pom.xml b/sdk/cosmos/azure-cosmos-spark_3/pom.xml index 72eee9af0f10..06eeb77f7282 100644 --- a/sdk/cosmos/azure-cosmos-spark_3/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark_3/pom.xml @@ -754,6 +754,9 @@ **/*Spec.* true + + true + diff --git a/sdk/cosmos/azure-cosmos-test/pom.xml b/sdk/cosmos/azure-cosmos-test/pom.xml index ee090944ee95..1a137181d078 100644 --- a/sdk/cosmos/azure-cosmos-test/pom.xml +++ b/sdk/cosmos/azure-cosmos-test/pom.xml @@ -79,7 +79,7 @@ Licensed under the MIT License. org.testng testng - 7.5.1 + 7.11.0 test @@ -162,6 +162,9 @@ Licensed under the MIT License. true + + true + diff --git a/sdk/cosmos/azure-cosmos-tests/pom.xml b/sdk/cosmos/azure-cosmos-tests/pom.xml index d02ae9502bdb..5fb36570236f 100644 --- a/sdk/cosmos/azure-cosmos-tests/pom.xml +++ b/sdk/cosmos/azure-cosmos-tests/pom.xml @@ -122,7 +122,7 @@ Licensed under the MIT License. org.testng testng - 7.5.1 + 7.11.0 test @@ -243,6 +243,9 @@ Licensed under the MIT License. true + + true + diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosAsyncClientTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosAsyncClientTest.java index 24fe8f9b06b6..72c2f1d43a04 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosAsyncClientTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosAsyncClientTest.java @@ -18,6 +18,7 @@ import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Listeners; import java.lang.management.BufferPoolMXBean; import java.lang.management.ManagementFactory; @@ -29,6 +30,7 @@ import static org.assertj.core.api.Assertions.assertThat; +@Listeners({TestNGLogListener.class, CosmosNettyLeakDetectorFactory.class}) public abstract class CosmosAsyncClientTest implements ITest { public static final String ROUTING_GATEWAY_EMULATOR_PORT = ":8081"; @@ -36,10 +38,8 @@ public abstract class CosmosAsyncClientTest implements ITest { protected static Logger logger = LoggerFactory.getLogger(CosmosAsyncClientTest.class.getSimpleName()); protected static final int SUITE_SETUP_TIMEOUT = 120000; - private final static AtomicInteger instancesUsed = new AtomicInteger(0); private final CosmosClientBuilder clientBuilder; private String testName; - private volatile Map activeClientsAtBegin = new HashMap<>(); public CosmosAsyncClientTest() { this(new CosmosClientBuilder()); @@ -49,81 +49,6 @@ public CosmosAsyncClientTest(CosmosClientBuilder clientBuilder) { this.clientBuilder = clientBuilder; } - @BeforeClass(groups = {"thinclient", "fast", "long", "direct", "multi-region", "multi-master", "flaky-multi-master", "emulator", - "emulator-vnext", "split", "query", "cfp-split", "circuit-breaker-misc-gateway", "circuit-breaker-misc-direct", - "circuit-breaker-read-all-read-many", "fi-multi-master", "long-emulator", "fi-thinclient-multi-region", "fi-thinclient-multi-master"}, timeOut = SUITE_SETUP_TIMEOUT, alwaysRun = true) - public void beforeClassSetupLeakDetection() { - if (instancesUsed.getAndIncrement() == 0) { - this.activeClientsAtBegin = RxDocumentClientImpl.getActiveClientsSnapshot(); - this.logMemoryUsage("BEFORE"); - } - } - - private void logMemoryUsage(String name) { - long pooledDirectBytes = PooledByteBufAllocator.DEFAULT.metric() - .directArenas().stream() - .mapToLong(io.netty.buffer.PoolArenaMetric::numActiveBytes) - .sum(); - - long used = PlatformDependent.usedDirectMemory(); - long max = PlatformDependent.maxDirectMemory(); - logger.info("MEMORY USAGE: {}:{}", this.getClass().getCanonicalName(), name); - logger.info("Netty Direct Memory: {}/{}/{} bytes", used, pooledDirectBytes, max); - for (BufferPoolMXBean pool : ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class)) { - logger.info("Pool {}: used={} bytes, capacity={} bytes, count={}", - pool.getName(), pool.getMemoryUsed(), pool.getTotalCapacity(), pool.getCount()); - } - } - - @AfterClass(groups = {"thinclient", "fast", "long", "direct", "multi-region", "multi-master", "flaky-multi-master", "emulator", - "emulator-vnext", "split", "query", "cfp-split", "circuit-breaker-misc-gateway", "circuit-breaker-misc-direct", - "circuit-breaker-read-all-read-many", "fi-multi-master", "long-emulator", "fi-thinclient-multi-region", "fi-thinclient-multi-master"}, timeOut = SUITE_SETUP_TIMEOUT, alwaysRun = true) - public void afterClassSetupLeakDetection() { - if (instancesUsed.decrementAndGet() == 0) { - Map leakedClientSnapshotNow = RxDocumentClientImpl.getActiveClientsSnapshot(); - StringBuilder sb = new StringBuilder(); - Map leakedClientSnapshotAtBegin = activeClientsAtBegin; - - for (Integer clientId : leakedClientSnapshotNow.keySet()) { - if (!leakedClientSnapshotAtBegin.containsKey(clientId)) { - // this client was leaked in this class - sb - .append("CosmosClient [") - .append(clientId) - .append("] leaked. Callstack of initialization:\n") - .append(leakedClientSnapshotNow.get(clientId)) - .append("\n\n"); - } - } - - if (sb.length() > 0) { - String msg = "COSMOS CLIENT LEAKS detected in test class: " - + this.getClass().getCanonicalName() - + "\n\n" - + sb; - - logger.error(msg); - // fail(msg); - } - - List nettyLeaks = CosmosNettyLeakDetectorFactory.resetIdentifiedLeaks(); - if (nettyLeaks.size() > 0) { - sb.append("\n"); - for (String leak : nettyLeaks) { - sb.append(leak).append("\n"); - } - - String msg = "NETTY LEAKS detected in test class: " - + this.getClass().getCanonicalName() - + sb; - - logger.error(msg); - // fail(msg); - } - this.logMemoryUsage("AFTER"); - } - } - public final CosmosClientBuilder getClientBuilder() { return this.clientBuilder; } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java index 1182209dc752..788dae937600 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java @@ -3,23 +3,36 @@ package com.azure.cosmos; import com.azure.cosmos.implementation.Configs; +import com.azure.cosmos.implementation.RxDocumentClientImpl; import com.azure.cosmos.implementation.StackTraceUtil; +import io.netty.buffer.PooledByteBufAllocator; import io.netty.util.ResourceLeakDetector; import io.netty.util.ResourceLeakDetectorFactory; +import io.netty.util.internal.PlatformDependent; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testng.IClassListener; import org.testng.IExecutionListener; +import org.testng.ITestClass; +import java.lang.management.BufferPoolMXBean; +import java.lang.management.ManagementFactory; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; + +public final class CosmosNettyLeakDetectorFactory + extends ResourceLeakDetectorFactory implements IExecutionListener, IClassListener { -public final class CosmosNettyLeakDetectorFactory extends ResourceLeakDetectorFactory implements IExecutionListener { protected static Logger logger = LoggerFactory.getLogger(CosmosNettyLeakDetectorFactory.class.getSimpleName()); private final static List identifiedLeaks = new ArrayList<>(); private final static Object staticLock = new Object(); private static volatile boolean isLeakDetectionDisabled = false; private static volatile boolean isInitialized = false; + private volatile Map activeClientsAtBegin = new HashMap<>(); + public CosmosNettyLeakDetectorFactory() { } @@ -29,16 +42,77 @@ public void onExecutionStart() { } @Override - public void onExecutionFinish() { - // Run GC to force finalizers to run - only in finalizers Netty would actually detect any leaks. - System.gc(); - try { - Thread.sleep(1_000); - } catch (InterruptedException e) { - throw new RuntimeException(e); + public void onBeforeClass(ITestClass testClass) { + if (Configs.isClientLeakDetectionEnabled()) { + this.activeClientsAtBegin = RxDocumentClientImpl.getActiveClientsSnapshot(); + this.logMemoryUsage("BEFORE CLASS", testClass.getRealClass().getCanonicalName()); + } + } + + @Override + public void onAfterClass(ITestClass testClass) { + if (Configs.isClientLeakDetectionEnabled()) { + Map leakedClientSnapshotNow = RxDocumentClientImpl.getActiveClientsSnapshot(); + StringBuilder sb = new StringBuilder(); + Map leakedClientSnapshotAtBegin = activeClientsAtBegin; + + for (Integer clientId : leakedClientSnapshotNow.keySet()) { + if (!leakedClientSnapshotAtBegin.containsKey(clientId)) { + // this client was leaked in this class + sb + .append("CosmosClient [") + .append(clientId) + .append("] leaked. Callstack of initialization:\n") + .append(leakedClientSnapshotNow.get(clientId)) + .append("\n\n"); + } + } + + if (sb.length() > 0) { + String msg = "COSMOS CLIENT LEAKS detected in test class: " + + testClass.getRealClass().getCanonicalName() + + "\n\n" + + sb; + + logger.error(msg); + // fail(msg); + } + + List nettyLeaks = CosmosNettyLeakDetectorFactory.resetIdentifiedLeaks(); + if (nettyLeaks.size() > 0) { + sb.append("\n"); + for (String leak : nettyLeaks) { + sb.append(leak).append("\n"); + } + + String msg = "NETTY LEAKS detected in test class: " + + this.getClass().getCanonicalName() + + sb; + + logger.error(msg); + // fail(msg); + } + this.logMemoryUsage("AFTER CLASS", testClass.getRealClass().getCanonicalName()); + } + } + + private void logMemoryUsage(String name, String className) { + long pooledDirectBytes = PooledByteBufAllocator.DEFAULT.metric() + .directArenas().stream() + .mapToLong(io.netty.buffer.PoolArenaMetric::numActiveBytes) + .sum(); + + long used = PlatformDependent.usedDirectMemory(); + long max = PlatformDependent.maxDirectMemory(); + logger.info("MEMORY USAGE: {}:{}", className, name); + logger.info("Netty Direct Memory: {}/{}/{} bytes", used, pooledDirectBytes, max); + for (BufferPoolMXBean pool : ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class)) { + logger.info("Pool {}: used={} bytes, capacity={} bytes, count={}", + pool.getName(), pool.getMemoryUsed(), pool.getTotalCapacity(), pool.getCount()); } } + // This method must be called as early as possible in the lifecycle of a process // before any Netty ByteBuf has been allocated public static void ingestIntoNetty() { diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/DocumentClientTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/DocumentClientTest.java index 000bfaffc0f0..44b609aa5a55 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/DocumentClientTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/DocumentClientTest.java @@ -15,6 +15,7 @@ import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Listeners; import java.lang.management.BufferPoolMXBean; import java.lang.management.ManagementFactory; @@ -24,13 +25,12 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; +@Listeners({TestNGLogListener.class, CosmosNettyLeakDetectorFactory.class}) public abstract class DocumentClientTest implements ITest { protected static Logger logger = LoggerFactory.getLogger(DocumentClientTest.class.getSimpleName()); protected static final int SUITE_SETUP_TIMEOUT = 120000; - private final static AtomicInteger instancesUsed = new AtomicInteger(0); private final AsyncDocumentClient.Builder clientBuilder; private String testName; - private volatile Map activeClientsAtBegin = new HashMap<>(); public DocumentClientTest() { this(new AsyncDocumentClient.Builder()); @@ -44,80 +44,6 @@ public final AsyncDocumentClient.Builder clientBuilder() { return this.clientBuilder; } - @BeforeClass(groups = {"fast", "long", "direct", "multi-region", "multi-master", "flaky-multi-master", "emulator", - "split", "query", "cfp-split", "long-emulator"}, timeOut = SUITE_SETUP_TIMEOUT) - - public void beforeClassSetupLeakDetection() { - if (instancesUsed.getAndIncrement() == 0) { - this.activeClientsAtBegin = RxDocumentClientImpl.getActiveClientsSnapshot(); - this.logMemoryUsage("BEFORE"); - } - } - - private void logMemoryUsage(String name) { - long pooledDirectBytes = PooledByteBufAllocator.DEFAULT.metric() - .directArenas().stream() - .mapToLong(io.netty.buffer.PoolArenaMetric::numActiveBytes) - .sum(); - - long used = PlatformDependent.usedDirectMemory(); - long max = PlatformDependent.maxDirectMemory(); - logger.info("MEMORY USAGE: {}:{}", this.getClass().getCanonicalName(), name); - logger.info("Netty Direct Memory: {}/{}/{} bytes", used, pooledDirectBytes, max); - for (BufferPoolMXBean pool : ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class)) { - logger.info("Pool {}: used={} bytes, capacity={} bytes, count={}", - pool.getName(), pool.getMemoryUsed(), pool.getTotalCapacity(), pool.getCount()); - } - } - - @AfterClass(groups = {"fast", "long", "direct", "multi-region", "multi-master", "flaky-multi-master", "emulator", - "split", "query", "cfp-split", "long-emulator"}, timeOut = SUITE_SETUP_TIMEOUT) - public void afterClassSetupLeakDetection() { - if (instancesUsed.decrementAndGet() == 0) { - Map leakedClientSnapshotNow = RxDocumentClientImpl.getActiveClientsSnapshot(); - StringBuilder sb = new StringBuilder(); - Map leakedClientSnapshotAtBegin = activeClientsAtBegin; - - for (Integer clientId : leakedClientSnapshotNow.keySet()) { - if (!leakedClientSnapshotAtBegin.containsKey(clientId)) { - // this client was leaked in this class - sb - .append("CosmosClient [") - .append(clientId) - .append("] leaked. Callstack of initialization:\n") - .append(leakedClientSnapshotNow.get(clientId)) - .append("\n\n"); - } - } - - if (sb.length() > 0) { - String msg = "COSMOS CLIENT LEAKS detected in test class: " - + this.getClass().getCanonicalName() - + "\n\n" - + sb; - - logger.error(msg); - // fail(msg); - } - - List nettyLeaks = CosmosNettyLeakDetectorFactory.resetIdentifiedLeaks(); - if (nettyLeaks.size() > 0) { - sb.append("\n"); - for (String leak : nettyLeaks) { - sb.append(leak).append("\n"); - } - - String msg = "NETTY LEAKS detected in test class: " - + this.getClass().getCanonicalName() - + sb; - - logger.error(msg); - // fail(msg); - } - this.logMemoryUsage("AFTER"); - } - } - @Override public final String getTestName() { return this.testName; diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestSuiteBase.java index 2208f14209aa..2a977e4159fd 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestSuiteBase.java @@ -55,8 +55,8 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.doAnswer; -@Listeners({TestNGLogListener.class}) -public class TestSuiteBase extends DocumentClientTest { +@Listeners({TestNGLogListener.class, CosmosNettyLeakDetectorFactory.class}) +public abstract class TestSuiteBase extends DocumentClientTest { private static final int DEFAULT_BULK_INSERT_CONCURRENCY_LEVEL = 500; private static final ObjectMapper objectMapper = new ObjectMapper(); protected static final int TIMEOUT = 40000; @@ -160,13 +160,6 @@ public void beforeSuite() { } } - @BeforeSuite(groups = {"unit"}) - public void parallelizeUnitTests(ITestContext context) { - // TODO: Parallelization was disabled due to flaky tests. Re-enable after fixing the flaky tests. -// context.getSuite().getXmlSuite().setParallel(XmlSuite.ParallelMode.CLASSES); -// context.getSuite().getXmlSuite().setThreadCount(Runtime.getRuntime().availableProcessors()); - } - @AfterSuite(groups = {"fast", "long", "direct", "multi-region", "multi-master", "flaky-multi-master", "emulator", "split", "query", "cfp-split", "long-emulator"}, timeOut = SUITE_SHUTDOWN_TIMEOUT) public void afterSuite() { diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java index 62582a4511a4..6691ef9ae322 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java @@ -96,8 +96,7 @@ import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.spy; -@Listeners({TestNGLogListener.class}) -public class TestSuiteBase extends CosmosAsyncClientTest { +public abstract class TestSuiteBase extends CosmosAsyncClientTest { private static final int DEFAULT_BULK_INSERT_CONCURRENCY_LEVEL = 500; private static final ObjectMapper objectMapper = new ObjectMapper(); @@ -222,13 +221,6 @@ public void beforeSuite() { } } - @BeforeSuite(groups = {"unit"}) - public void parallelizeUnitTests(ITestContext context) { - // TODO: Parallelization was disabled due to flaky tests. Re-enable after fixing the flaky tests. -// context.getSuite().getXmlSuite().setParallel(XmlSuite.ParallelMode.CLASSES); -// context.getSuite().getXmlSuite().setThreadCount(Runtime.getRuntime().availableProcessors()); - } - @AfterSuite(groups = {"thinclient", "fast", "long", "direct", "multi-region", "multi-master", "flaky-multi-master", "emulator", "split", "query", "cfp-split", "circuit-breaker-misc-gateway", "circuit-breaker-misc-direct", "circuit-breaker-read-all-read-many", "fi-multi-master", "long-emulator", "fi-thinclient-multi-region", "fi-thinclient-multi-master"}, timeOut = SUITE_SHUTDOWN_TIMEOUT) diff --git a/sdk/cosmos/azure-cosmos/pom.xml b/sdk/cosmos/azure-cosmos/pom.xml index b474bbe5f73b..2cc50fad0240 100644 --- a/sdk/cosmos/azure-cosmos/pom.xml +++ b/sdk/cosmos/azure-cosmos/pom.xml @@ -110,7 +110,7 @@ Licensed under the MIT License. org.testng testng - 7.5.1 + 7.11.0 test diff --git a/sdk/cosmos/fabric-cosmos-spark-auth_3/pom.xml b/sdk/cosmos/fabric-cosmos-spark-auth_3/pom.xml index 2941fa16041d..ecaa7ba5cdd5 100644 --- a/sdk/cosmos/fabric-cosmos-spark-auth_3/pom.xml +++ b/sdk/cosmos/fabric-cosmos-spark-auth_3/pom.xml @@ -513,6 +513,9 @@ **/*Spec.* true + + true + From c60f309166ed4d77ffbcd120d42b3ba3f1d2b631 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Tue, 11 Nov 2025 18:15:05 +0000 Subject: [PATCH 31/65] Reverting TestNG to 7.9.0 (highest version still supporting Java8) --- eng/versioning/external_dependencies.txt | 2 +- sdk/cosmos/azure-cosmos-benchmark/pom.xml | 2 +- sdk/cosmos/azure-cosmos-encryption/pom.xml | 2 +- sdk/cosmos/azure-cosmos-kafka-connect/pom.xml | 2 +- sdk/cosmos/azure-cosmos-test/pom.xml | 2 +- sdk/cosmos/azure-cosmos-tests/pom.xml | 2 +- sdk/cosmos/azure-cosmos/pom.xml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/eng/versioning/external_dependencies.txt b/eng/versioning/external_dependencies.txt index 09453aa8bebb..586d43f3aaf4 100644 --- a/eng/versioning/external_dependencies.txt +++ b/eng/versioning/external_dependencies.txt @@ -152,7 +152,7 @@ org.junit.jupiter:junit-jupiter-params;5.13.4 org.junit.platform:junit-platform-launcher;1.13.4 org.openjdk.jmh:jmh-core;1.37 org.openjdk.jmh:jmh-generator-annprocess;1.37 -org.testng:testng;7.11.0 +org.testng:testng;7.9.0 uk.org.webcompere:system-stubs-jupiter;2.0.2 com.google.truth:truth;1.1.3 diff --git a/sdk/cosmos/azure-cosmos-benchmark/pom.xml b/sdk/cosmos/azure-cosmos-benchmark/pom.xml index cd7b43b5e9d1..372fa5fa55de 100644 --- a/sdk/cosmos/azure-cosmos-benchmark/pom.xml +++ b/sdk/cosmos/azure-cosmos-benchmark/pom.xml @@ -165,7 +165,7 @@ Licensed under the MIT License. org.testng testng - 7.11.0 + 7.9.0 test diff --git a/sdk/cosmos/azure-cosmos-encryption/pom.xml b/sdk/cosmos/azure-cosmos-encryption/pom.xml index 584758735f6a..b983a5bf914f 100644 --- a/sdk/cosmos/azure-cosmos-encryption/pom.xml +++ b/sdk/cosmos/azure-cosmos-encryption/pom.xml @@ -130,7 +130,7 @@ Licensed under the MIT License. org.testng testng - 7.11.0 + 7.9.0 test diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml b/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml index 6e3dfe5465ec..4d25bc1df501 100644 --- a/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml +++ b/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml @@ -160,7 +160,7 @@ Licensed under the MIT License. org.testng testng - 7.11.0 + 7.9.0 test diff --git a/sdk/cosmos/azure-cosmos-test/pom.xml b/sdk/cosmos/azure-cosmos-test/pom.xml index 1a137181d078..dd86e7745df4 100644 --- a/sdk/cosmos/azure-cosmos-test/pom.xml +++ b/sdk/cosmos/azure-cosmos-test/pom.xml @@ -79,7 +79,7 @@ Licensed under the MIT License. org.testng testng - 7.11.0 + 7.9.0 test diff --git a/sdk/cosmos/azure-cosmos-tests/pom.xml b/sdk/cosmos/azure-cosmos-tests/pom.xml index 5fb36570236f..ba2932f6d3ea 100644 --- a/sdk/cosmos/azure-cosmos-tests/pom.xml +++ b/sdk/cosmos/azure-cosmos-tests/pom.xml @@ -122,7 +122,7 @@ Licensed under the MIT License. org.testng testng - 7.11.0 + 7.9.0 test diff --git a/sdk/cosmos/azure-cosmos/pom.xml b/sdk/cosmos/azure-cosmos/pom.xml index 2cc50fad0240..e17df3283969 100644 --- a/sdk/cosmos/azure-cosmos/pom.xml +++ b/sdk/cosmos/azure-cosmos/pom.xml @@ -110,7 +110,7 @@ Licensed under the MIT License. org.testng testng - 7.11.0 + 7.9.0 test From f9ecf9e2a00ca42eb8bd5dca009649063a27d27b Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Tue, 11 Nov 2025 20:33:51 +0000 Subject: [PATCH 32/65] Switching back to TestNG 7.5.1 --- eng/versioning/external_dependencies.txt | 2 +- sdk/cosmos/azure-cosmos-benchmark/pom.xml | 5 +- sdk/cosmos/azure-cosmos-encryption/pom.xml | 5 +- .../CosmosNettyLeakDetectorFactory.java | 154 +++++++++++++++++- sdk/cosmos/azure-cosmos-kafka-connect/pom.xml | 5 +- .../pom.xml | 3 - .../azure-cosmos-spark_3-3_2-12/pom.xml | 3 - .../azure-cosmos-spark_3-4_2-12/pom.xml | 3 - .../azure-cosmos-spark_3-5_2-12/pom.xml | 3 - sdk/cosmos/azure-cosmos-spark_3/pom.xml | 3 - sdk/cosmos/azure-cosmos-test/pom.xml | 5 +- sdk/cosmos/azure-cosmos-tests/pom.xml | 5 +- .../CosmosNettyLeakDetectorFactory.java | 132 +++++++++++---- sdk/cosmos/azure-cosmos/pom.xml | 2 +- sdk/cosmos/fabric-cosmos-spark-auth_3/pom.xml | 3 - 15 files changed, 250 insertions(+), 83 deletions(-) diff --git a/eng/versioning/external_dependencies.txt b/eng/versioning/external_dependencies.txt index 586d43f3aaf4..47e532c6ee5c 100644 --- a/eng/versioning/external_dependencies.txt +++ b/eng/versioning/external_dependencies.txt @@ -152,7 +152,7 @@ org.junit.jupiter:junit-jupiter-params;5.13.4 org.junit.platform:junit-platform-launcher;1.13.4 org.openjdk.jmh:jmh-core;1.37 org.openjdk.jmh:jmh-generator-annprocess;1.37 -org.testng:testng;7.9.0 +org.testng:testng;7.5.1 uk.org.webcompere:system-stubs-jupiter;2.0.2 com.google.truth:truth;1.1.3 diff --git a/sdk/cosmos/azure-cosmos-benchmark/pom.xml b/sdk/cosmos/azure-cosmos-benchmark/pom.xml index 372fa5fa55de..7ae0b50faafb 100644 --- a/sdk/cosmos/azure-cosmos-benchmark/pom.xml +++ b/sdk/cosmos/azure-cosmos-benchmark/pom.xml @@ -165,7 +165,7 @@ Licensed under the MIT License. org.testng testng - 7.9.0 + 7.5.1 test @@ -213,9 +213,6 @@ Licensed under the MIT License. true - - true - diff --git a/sdk/cosmos/azure-cosmos-encryption/pom.xml b/sdk/cosmos/azure-cosmos-encryption/pom.xml index b983a5bf914f..a9984111c017 100644 --- a/sdk/cosmos/azure-cosmos-encryption/pom.xml +++ b/sdk/cosmos/azure-cosmos-encryption/pom.xml @@ -130,7 +130,7 @@ Licensed under the MIT License. org.testng testng - 7.9.0 + 7.5.1 test @@ -227,9 +227,6 @@ Licensed under the MIT License. true - - true - diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java index 8fcd4f418548..42f42a78102e 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java @@ -3,23 +3,42 @@ package com.azure.cosmos.encryption; import com.azure.cosmos.implementation.Configs; +import com.azure.cosmos.implementation.RxDocumentClientImpl; import com.azure.cosmos.implementation.StackTraceUtil; +import io.netty.buffer.PooledByteBufAllocator; import io.netty.util.ResourceLeakDetector; import io.netty.util.ResourceLeakDetectorFactory; +import io.netty.util.internal.PlatformDependent; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testng.IClassListener; import org.testng.IExecutionListener; +import org.testng.IInvokedMethod; +import org.testng.IInvokedMethodListener; +import org.testng.ITestClass; +import org.testng.ITestResult; +import java.lang.management.BufferPoolMXBean; +import java.lang.management.ManagementFactory; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +public final class CosmosNettyLeakDetectorFactory + extends ResourceLeakDetectorFactory implements IExecutionListener, IInvokedMethodListener, IClassListener { -public final class CosmosNettyLeakDetectorFactory extends ResourceLeakDetectorFactory implements IExecutionListener { protected static Logger logger = LoggerFactory.getLogger(CosmosNettyLeakDetectorFactory.class.getSimpleName()); private final static List identifiedLeaks = new ArrayList<>(); private final static Object staticLock = new Object(); + + private final static Map testClassInventory = new HashMap<>(); private static volatile boolean isLeakDetectionDisabled = false; private static volatile boolean isInitialized = false; + private volatile Map activeClientsAtBegin = new HashMap<>(); + public CosmosNettyLeakDetectorFactory() { } @@ -29,16 +48,133 @@ public void onExecutionStart() { } @Override - public void onExecutionFinish() { - // Run GC to force finalizers to run - only in finalizers Netty would actually detect any leaks. - System.gc(); - try { - Thread.sleep(1_000); - } catch (InterruptedException e) { - throw new RuntimeException(e); + public void onBeforeClass(ITestClass testClass) { + if (Configs.isClientLeakDetectionEnabled()) { + String testClassName = testClass.getRealClass().getCanonicalName(); + AtomicInteger instanceCountSnapshot = null; + synchronized (staticLock) { + instanceCountSnapshot = testClassInventory.get(testClassName); + if (instanceCountSnapshot == null) { + testClassInventory.put(testClassName, instanceCountSnapshot = new AtomicInteger(0)); + } + } + + int alreadyInitializedInstanceCount = instanceCountSnapshot.getAndIncrement(); + if (alreadyInitializedInstanceCount == 0) { + logger.info("LEAK DETECTION INITIALIZATION for test class {}", testClassName); + this.activeClientsAtBegin = RxDocumentClientImpl.getActiveClientsSnapshot(); + this.logMemoryUsage("BEFORE CLASS", testClassName); + } + } + } + + @Override + public void onAfterClass(ITestClass testClass) { + // Unfortunately can't use this in TestNG 7.51 because execution is not symmetric + // IClassListener.onBeforeClass + // TestClassBase.@BeforeClass + // TestClass.@BeforeClass + // IClassListener.onAfterClass + // TestClass.@AfterClass + // TestClassBase.@AfterClass + // we would want the IClassListener.onAfterClass to be called last - which system property + // -Dtestng.listener.execution.symmetric=true allows, but this is only available + // in TestNG 7.7.1 - which requires Java11 + // So, this class simulates this behavior by hooking into IInvokedMethodListener + } + + @Override + public void afterInvocation(IInvokedMethod method, ITestResult result) { + if (method.isConfigurationMethod() + && method.getTestMethod().isAfterClassConfiguration()) { + // <-- This point is guaranteed to be AFTER the class’s @AfterClass ran + ITestClass testClass = (ITestClass)result.getTestClass(); + try { + this.onAfterClassCore(testClass); + } catch (Throwable t) { + // decide if you want to fail the suite or just log + System.err.println("Symmetric afterClass failed for " + testClass.getRealClass().getCanonicalName()); + t.printStackTrace(); + } + } + } + + private void onAfterClassCore(ITestClass testClass) { + if (Configs.isClientLeakDetectionEnabled()) { + String testClassName = testClass.getRealClass().getCanonicalName(); + AtomicInteger instanceCountSnapshot = null; + synchronized (staticLock) { + instanceCountSnapshot = testClassInventory.get(testClassName); + if (instanceCountSnapshot == null) { + testClassInventory.put(testClassName, instanceCountSnapshot = new AtomicInteger(0)); + } + } + + int remainingInstanceCount = instanceCountSnapshot.decrementAndGet(); + if (remainingInstanceCount == 0) { + logger.info("LEAK DETECTION EVALUATION for test class {}", testClassName); + Map leakedClientSnapshotNow = RxDocumentClientImpl.getActiveClientsSnapshot(); + StringBuilder sb = new StringBuilder(); + Map leakedClientSnapshotAtBegin = activeClientsAtBegin; + + for (Integer clientId : leakedClientSnapshotNow.keySet()) { + if (!leakedClientSnapshotAtBegin.containsKey(clientId)) { + // this client was leaked in this class + sb + .append("CosmosClient [") + .append(clientId) + .append("] leaked. Callstack of initialization:\n") + .append(leakedClientSnapshotNow.get(clientId)) + .append("\n\n"); + } + } + + if (sb.length() > 0) { + String msg = "COSMOS CLIENT LEAKS detected in test class: " + + testClassName + + "\n\n" + + sb; + + logger.error(msg); + // fail(msg); + } + + List nettyLeaks = CosmosNettyLeakDetectorFactory.resetIdentifiedLeaks(); + if (nettyLeaks.size() > 0) { + sb.append("\n"); + for (String leak : nettyLeaks) { + sb.append(leak).append("\n"); + } + + String msg = "NETTY LEAKS detected in test class: " + + this.getClass().getCanonicalName() + + sb; + + logger.error(msg); + // fail(msg); + } + this.logMemoryUsage("AFTER CLASS", testClassName); + } + } + } + + private void logMemoryUsage(String name, String className) { + long pooledDirectBytes = PooledByteBufAllocator.DEFAULT.metric() + .directArenas().stream() + .mapToLong(io.netty.buffer.PoolArenaMetric::numActiveBytes) + .sum(); + + long used = PlatformDependent.usedDirectMemory(); + long max = PlatformDependent.maxDirectMemory(); + logger.info("MEMORY USAGE: {}:{}", className, name); + logger.info("Netty Direct Memory: {}/{}/{} bytes", used, pooledDirectBytes, max); + for (BufferPoolMXBean pool : ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class)) { + logger.info("Pool {}: used={} bytes, capacity={} bytes, count={}", + pool.getName(), pool.getMemoryUsed(), pool.getTotalCapacity(), pool.getCount()); } } + // This method must be called as early as possible in the lifecycle of a process // before any Netty ByteBuf has been allocated public static void ingestIntoNetty() { @@ -134,8 +270,10 @@ protected void reportInstancesLeak(String resourceType) { synchronized (staticLock) { if (!isLeakDetectionDisabled) { String msg = "NETTY LEAK (instances) type=" + resourceType; + identifiedLeaks.add(msg); logger.error(msg); + } } } diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml b/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml index 4d25bc1df501..1693af8dc8fb 100644 --- a/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml +++ b/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml @@ -160,7 +160,7 @@ Licensed under the MIT License. org.testng testng - 7.9.0 + 7.5.1 test @@ -286,9 +286,6 @@ Licensed under the MIT License. true - - true - diff --git a/sdk/cosmos/azure-cosmos-spark-account-data-resolver-sample/pom.xml b/sdk/cosmos/azure-cosmos-spark-account-data-resolver-sample/pom.xml index 9951686b5b27..7391f7308117 100644 --- a/sdk/cosmos/azure-cosmos-spark-account-data-resolver-sample/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark-account-data-resolver-sample/pom.xml @@ -703,9 +703,6 @@ **/*Spec.* true - - true - diff --git a/sdk/cosmos/azure-cosmos-spark_3-3_2-12/pom.xml b/sdk/cosmos/azure-cosmos-spark_3-3_2-12/pom.xml index 7a7d299de19f..948990bafe7f 100644 --- a/sdk/cosmos/azure-cosmos-spark_3-3_2-12/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark_3-3_2-12/pom.xml @@ -125,9 +125,6 @@ **/*Spec.* true - - true - diff --git a/sdk/cosmos/azure-cosmos-spark_3-4_2-12/pom.xml b/sdk/cosmos/azure-cosmos-spark_3-4_2-12/pom.xml index 98483ad3ad1b..fcecfd210ea5 100644 --- a/sdk/cosmos/azure-cosmos-spark_3-4_2-12/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark_3-4_2-12/pom.xml @@ -125,9 +125,6 @@ **/*Spec.* true - - true - diff --git a/sdk/cosmos/azure-cosmos-spark_3-5_2-12/pom.xml b/sdk/cosmos/azure-cosmos-spark_3-5_2-12/pom.xml index b2eff8159e95..435e9fd672e5 100644 --- a/sdk/cosmos/azure-cosmos-spark_3-5_2-12/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark_3-5_2-12/pom.xml @@ -125,9 +125,6 @@ **/*Spec.* true - - true - diff --git a/sdk/cosmos/azure-cosmos-spark_3/pom.xml b/sdk/cosmos/azure-cosmos-spark_3/pom.xml index 06eeb77f7282..72eee9af0f10 100644 --- a/sdk/cosmos/azure-cosmos-spark_3/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark_3/pom.xml @@ -754,9 +754,6 @@ **/*Spec.* true - - true - diff --git a/sdk/cosmos/azure-cosmos-test/pom.xml b/sdk/cosmos/azure-cosmos-test/pom.xml index dd86e7745df4..ee090944ee95 100644 --- a/sdk/cosmos/azure-cosmos-test/pom.xml +++ b/sdk/cosmos/azure-cosmos-test/pom.xml @@ -79,7 +79,7 @@ Licensed under the MIT License. org.testng testng - 7.9.0 + 7.5.1 test @@ -162,9 +162,6 @@ Licensed under the MIT License. true - - true - diff --git a/sdk/cosmos/azure-cosmos-tests/pom.xml b/sdk/cosmos/azure-cosmos-tests/pom.xml index ba2932f6d3ea..d02ae9502bdb 100644 --- a/sdk/cosmos/azure-cosmos-tests/pom.xml +++ b/sdk/cosmos/azure-cosmos-tests/pom.xml @@ -122,7 +122,7 @@ Licensed under the MIT License. org.testng testng - 7.9.0 + 7.5.1 test @@ -243,9 +243,6 @@ Licensed under the MIT License. true - - true - diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java index 788dae937600..9f9a6f071889 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java @@ -13,7 +13,10 @@ import org.slf4j.LoggerFactory; import org.testng.IClassListener; import org.testng.IExecutionListener; +import org.testng.IInvokedMethod; +import org.testng.IInvokedMethodListener; import org.testng.ITestClass; +import org.testng.ITestResult; import java.lang.management.BufferPoolMXBean; import java.lang.management.ManagementFactory; @@ -21,13 +24,16 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; public final class CosmosNettyLeakDetectorFactory - extends ResourceLeakDetectorFactory implements IExecutionListener, IClassListener { + extends ResourceLeakDetectorFactory implements IExecutionListener, IInvokedMethodListener, IClassListener { protected static Logger logger = LoggerFactory.getLogger(CosmosNettyLeakDetectorFactory.class.getSimpleName()); private final static List identifiedLeaks = new ArrayList<>(); private final static Object staticLock = new Object(); + + private final static Map testClassInventory = new HashMap<>(); private static volatile boolean isLeakDetectionDisabled = false; private static volatile boolean isInitialized = false; @@ -44,55 +50,111 @@ public void onExecutionStart() { @Override public void onBeforeClass(ITestClass testClass) { if (Configs.isClientLeakDetectionEnabled()) { - this.activeClientsAtBegin = RxDocumentClientImpl.getActiveClientsSnapshot(); - this.logMemoryUsage("BEFORE CLASS", testClass.getRealClass().getCanonicalName()); + String testClassName = testClass.getRealClass().getCanonicalName(); + AtomicInteger instanceCountSnapshot = null; + synchronized (staticLock) { + instanceCountSnapshot = testClassInventory.get(testClassName); + if (instanceCountSnapshot == null) { + testClassInventory.put(testClassName, instanceCountSnapshot = new AtomicInteger(0)); + } + } + + int alreadyInitializedInstanceCount = instanceCountSnapshot.getAndIncrement(); + if (alreadyInitializedInstanceCount == 0) { + logger.info("LEAK DETECTION INITIALIZATION for test class {}", testClassName); + this.activeClientsAtBegin = RxDocumentClientImpl.getActiveClientsSnapshot(); + this.logMemoryUsage("BEFORE CLASS", testClassName); + } } } @Override public void onAfterClass(ITestClass testClass) { + // Unfortunately can't use this in TestNG 7.51 because execution is not symmetric + // IClassListener.onBeforeClass + // TestClassBase.@BeforeClass + // TestClass.@BeforeClass + // IClassListener.onAfterClass + // TestClass.@AfterClass + // TestClassBase.@AfterClass + // we would want the IClassListener.onAfterClass to be called last - which system property + // -Dtestng.listener.execution.symmetric=true allows, but this is only available + // in TestNG 7.7.1 - which requires Java11 + // So, this class simulates this behavior by hooking into IInvokedMethodListener + } + + @Override + public void afterInvocation(IInvokedMethod method, ITestResult result) { + if (method.isConfigurationMethod() + && method.getTestMethod().isAfterClassConfiguration()) { + // <-- This point is guaranteed to be AFTER the class’s @AfterClass ran + ITestClass testClass = (ITestClass)result.getTestClass(); + try { + this.onAfterClassCore(testClass); + } catch (Throwable t) { + // decide if you want to fail the suite or just log + System.err.println("Symmetric afterClass failed for " + testClass.getRealClass().getCanonicalName()); + t.printStackTrace(); + } + } + } + + private void onAfterClassCore(ITestClass testClass) { if (Configs.isClientLeakDetectionEnabled()) { - Map leakedClientSnapshotNow = RxDocumentClientImpl.getActiveClientsSnapshot(); - StringBuilder sb = new StringBuilder(); - Map leakedClientSnapshotAtBegin = activeClientsAtBegin; - - for (Integer clientId : leakedClientSnapshotNow.keySet()) { - if (!leakedClientSnapshotAtBegin.containsKey(clientId)) { - // this client was leaked in this class - sb - .append("CosmosClient [") - .append(clientId) - .append("] leaked. Callstack of initialization:\n") - .append(leakedClientSnapshotNow.get(clientId)) - .append("\n\n"); + String testClassName = testClass.getRealClass().getCanonicalName(); + AtomicInteger instanceCountSnapshot = null; + synchronized (staticLock) { + instanceCountSnapshot = testClassInventory.get(testClassName); + if (instanceCountSnapshot == null) { + testClassInventory.put(testClassName, instanceCountSnapshot = new AtomicInteger(0)); } } - if (sb.length() > 0) { - String msg = "COSMOS CLIENT LEAKS detected in test class: " - + testClass.getRealClass().getCanonicalName() - + "\n\n" - + sb; + int remainingInstanceCount = instanceCountSnapshot.decrementAndGet(); + if (remainingInstanceCount == 0) { + logger.info("LEAK DETECTION EVALUATION for test class {}", testClassName); + Map leakedClientSnapshotNow = RxDocumentClientImpl.getActiveClientsSnapshot(); + StringBuilder sb = new StringBuilder(); + Map leakedClientSnapshotAtBegin = activeClientsAtBegin; + + for (Integer clientId : leakedClientSnapshotNow.keySet()) { + if (!leakedClientSnapshotAtBegin.containsKey(clientId)) { + // this client was leaked in this class + sb + .append("CosmosClient [") + .append(clientId) + .append("] leaked. Callstack of initialization:\n") + .append(leakedClientSnapshotNow.get(clientId)) + .append("\n\n"); + } + } - logger.error(msg); - // fail(msg); - } + if (sb.length() > 0) { + String msg = "COSMOS CLIENT LEAKS detected in test class: " + + testClassName + + "\n\n" + + sb; - List nettyLeaks = CosmosNettyLeakDetectorFactory.resetIdentifiedLeaks(); - if (nettyLeaks.size() > 0) { - sb.append("\n"); - for (String leak : nettyLeaks) { - sb.append(leak).append("\n"); + logger.error(msg); + // fail(msg); } - String msg = "NETTY LEAKS detected in test class: " - + this.getClass().getCanonicalName() - + sb; + List nettyLeaks = CosmosNettyLeakDetectorFactory.resetIdentifiedLeaks(); + if (nettyLeaks.size() > 0) { + sb.append("\n"); + for (String leak : nettyLeaks) { + sb.append(leak).append("\n"); + } - logger.error(msg); - // fail(msg); + String msg = "NETTY LEAKS detected in test class: " + + this.getClass().getCanonicalName() + + sb; + + logger.error(msg); + // fail(msg); + } + this.logMemoryUsage("AFTER CLASS", testClassName); } - this.logMemoryUsage("AFTER CLASS", testClass.getRealClass().getCanonicalName()); } } diff --git a/sdk/cosmos/azure-cosmos/pom.xml b/sdk/cosmos/azure-cosmos/pom.xml index e17df3283969..b474bbe5f73b 100644 --- a/sdk/cosmos/azure-cosmos/pom.xml +++ b/sdk/cosmos/azure-cosmos/pom.xml @@ -110,7 +110,7 @@ Licensed under the MIT License. org.testng testng - 7.9.0 + 7.5.1 test diff --git a/sdk/cosmos/fabric-cosmos-spark-auth_3/pom.xml b/sdk/cosmos/fabric-cosmos-spark-auth_3/pom.xml index ecaa7ba5cdd5..2941fa16041d 100644 --- a/sdk/cosmos/fabric-cosmos-spark-auth_3/pom.xml +++ b/sdk/cosmos/fabric-cosmos-spark-auth_3/pom.xml @@ -513,9 +513,6 @@ **/*Spec.* true - - true - From f71d7b57198c16cd525aed6077b4d74ea2306686 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Tue, 11 Nov 2025 21:52:30 +0000 Subject: [PATCH 33/65] Enabling leak detection in unit tests --- sdk/cosmos/azure-cosmos-benchmark/pom.xml | 42 +++++++++ sdk/cosmos/azure-cosmos-encryption/pom.xml | 10 ++- sdk/cosmos/azure-cosmos-kafka-connect/pom.xml | 19 +++- .../pom.xml | 3 + .../azure-cosmos-spark_3-3_2-12/pom.xml | 3 + .../azure-cosmos-spark_3-4_2-12/pom.xml | 3 + .../azure-cosmos-spark_3-5_2-12/pom.xml | 3 + sdk/cosmos/azure-cosmos-spark_3/pom.xml | 3 + sdk/cosmos/azure-cosmos-test/pom.xml | 3 + sdk/cosmos/azure-cosmos-tests/pom.xml | 89 ++++++++++++++----- sdk/cosmos/azure-cosmos/pom.xml | 3 + sdk/cosmos/fabric-cosmos-spark-auth_3/pom.xml | 3 + 12 files changed, 159 insertions(+), 25 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-benchmark/pom.xml b/sdk/cosmos/azure-cosmos-benchmark/pom.xml index 7ae0b50faafb..33c3f11d8292 100644 --- a/sdk/cosmos/azure-cosmos-benchmark/pom.xml +++ b/sdk/cosmos/azure-cosmos-benchmark/pom.xml @@ -203,6 +203,9 @@ Licensed under the MIT License. 3.5.3 unit + + true + %regex[.*] @@ -308,6 +311,9 @@ Licensed under the MIT License. src/test/resources/unit-testng.xml + + true + @@ -329,6 +335,9 @@ Licensed under the MIT License. src/test/resources/fast-testng.xml + + true + @@ -350,6 +359,9 @@ Licensed under the MIT License. src/test/resources/split-testng.xml + + true + @@ -371,6 +383,9 @@ Licensed under the MIT License. src/test/resources/cfp-split-testng.xml + + true + @@ -392,6 +407,9 @@ Licensed under the MIT License. src/test/resources/query-testng.xml + + true + @@ -413,6 +431,9 @@ Licensed under the MIT License. src/test/resources/long-testng.xml + + true + @@ -434,6 +455,9 @@ Licensed under the MIT License. src/test/resources/direct-testng.xml + + true + @@ -455,6 +479,9 @@ Licensed under the MIT License. src/test/resources/multi-master-testng.xml + + true + @@ -476,6 +503,9 @@ Licensed under the MIT License. src/test/resources/flaky-multi-master-testng.xml + + true + @@ -497,6 +527,9 @@ Licensed under the MIT License. src/test/resources/fi-multi-master-testng.xml + + true + @@ -519,6 +552,9 @@ Licensed under the MIT License. src/test/resources/examples-testng.xml + + true + @@ -548,6 +584,9 @@ Licensed under the MIT License. src/test/resources/emulator-testng.xml + + true + @@ -569,6 +608,9 @@ Licensed under the MIT License. src/test/resources/e2e-testng.xml + + true + diff --git a/sdk/cosmos/azure-cosmos-encryption/pom.xml b/sdk/cosmos/azure-cosmos-encryption/pom.xml index a9984111c017..1bcea530e743 100644 --- a/sdk/cosmos/azure-cosmos-encryption/pom.xml +++ b/sdk/cosmos/azure-cosmos-encryption/pom.xml @@ -217,6 +217,9 @@ Licensed under the MIT License. 3.5.3 unit + + true + %regex[.*] @@ -263,6 +266,9 @@ Licensed under the MIT License. src/test/resources/unit-testng.xml + + true + @@ -284,7 +290,9 @@ Licensed under the MIT License. src/test/resources/encryption-testng.xml - + + true + diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml b/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml index 1693af8dc8fb..9b5fc2d64295 100644 --- a/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml +++ b/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml @@ -276,6 +276,9 @@ Licensed under the MIT License. 3.5.3 unit + + true + %regex[.*] @@ -533,7 +536,9 @@ Licensed under the MIT License. src/test/resources/unit-testng.xml - + + true + @@ -555,7 +560,9 @@ Licensed under the MIT License. src/test/resources/kafka-emulator-testng.xml - + + true + @@ -577,7 +584,9 @@ Licensed under the MIT License. src/test/resources/kafka-testng.xml - + + true + @@ -599,7 +608,9 @@ Licensed under the MIT License. src/test/resources/kafka-integration-testng.xml - + + true + diff --git a/sdk/cosmos/azure-cosmos-spark-account-data-resolver-sample/pom.xml b/sdk/cosmos/azure-cosmos-spark-account-data-resolver-sample/pom.xml index 7391f7308117..5025828b0ae1 100644 --- a/sdk/cosmos/azure-cosmos-spark-account-data-resolver-sample/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark-account-data-resolver-sample/pom.xml @@ -696,6 +696,9 @@ maven-surefire-plugin 3.5.3 + + true + **/*.* **/*Test.* diff --git a/sdk/cosmos/azure-cosmos-spark_3-3_2-12/pom.xml b/sdk/cosmos/azure-cosmos-spark_3-3_2-12/pom.xml index 948990bafe7f..9e4c51da4257 100644 --- a/sdk/cosmos/azure-cosmos-spark_3-3_2-12/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark_3-3_2-12/pom.xml @@ -118,6 +118,9 @@ maven-surefire-plugin 3.5.3 + + true + **/*.* **/*Test.* diff --git a/sdk/cosmos/azure-cosmos-spark_3-4_2-12/pom.xml b/sdk/cosmos/azure-cosmos-spark_3-4_2-12/pom.xml index fcecfd210ea5..ed6decd27996 100644 --- a/sdk/cosmos/azure-cosmos-spark_3-4_2-12/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark_3-4_2-12/pom.xml @@ -118,6 +118,9 @@ maven-surefire-plugin 3.5.3 + + true + **/*.* **/*Test.* diff --git a/sdk/cosmos/azure-cosmos-spark_3-5_2-12/pom.xml b/sdk/cosmos/azure-cosmos-spark_3-5_2-12/pom.xml index 435e9fd672e5..1026b640d170 100644 --- a/sdk/cosmos/azure-cosmos-spark_3-5_2-12/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark_3-5_2-12/pom.xml @@ -118,6 +118,9 @@ maven-surefire-plugin 3.5.3 + + true + **/*.* **/*Test.* diff --git a/sdk/cosmos/azure-cosmos-spark_3/pom.xml b/sdk/cosmos/azure-cosmos-spark_3/pom.xml index 72eee9af0f10..b241bba414c7 100644 --- a/sdk/cosmos/azure-cosmos-spark_3/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark_3/pom.xml @@ -747,6 +747,9 @@ maven-surefire-plugin 3.5.3 + + true + **/*.* **/*Test.* diff --git a/sdk/cosmos/azure-cosmos-test/pom.xml b/sdk/cosmos/azure-cosmos-test/pom.xml index ee090944ee95..38566c8e94a3 100644 --- a/sdk/cosmos/azure-cosmos-test/pom.xml +++ b/sdk/cosmos/azure-cosmos-test/pom.xml @@ -152,6 +152,9 @@ Licensed under the MIT License. 3.5.3 unit + + true + %regex[.*] diff --git a/sdk/cosmos/azure-cosmos-tests/pom.xml b/sdk/cosmos/azure-cosmos-tests/pom.xml index d02ae9502bdb..8ae1a907833e 100644 --- a/sdk/cosmos/azure-cosmos-tests/pom.xml +++ b/sdk/cosmos/azure-cosmos-tests/pom.xml @@ -233,6 +233,9 @@ Licensed under the MIT License. 3.5.3 unit + + true + %regex[.*] @@ -314,7 +317,9 @@ Licensed under the MIT License. src/test/resources/unit-testng.xml - + + true + @@ -336,7 +341,9 @@ Licensed under the MIT License. src/test/resources/fast-testng.xml - + + true + @@ -358,7 +365,9 @@ Licensed under the MIT License. src/test/resources/split-testng.xml - + + true + @@ -380,7 +389,9 @@ Licensed under the MIT License. src/test/resources/cfp-split-testng.xml - + + true + @@ -402,7 +413,9 @@ Licensed under the MIT License. src/test/resources/query-testng.xml - + + true + @@ -424,7 +437,9 @@ Licensed under the MIT License. src/test/resources/long-testng.xml - + + true + @@ -446,7 +461,9 @@ Licensed under the MIT License. src/test/resources/direct-testng.xml - + + true + @@ -468,7 +485,9 @@ Licensed under the MIT License. src/test/resources/multi-master-testng.xml - + + true + @@ -490,7 +509,9 @@ Licensed under the MIT License. src/test/resources/circuit-breaker-read-all-read-many-testng.xml - + + true + @@ -512,7 +533,9 @@ Licensed under the MIT License. src/test/resources/circuit-breaker-misc-direct-testng.xml - + + true + @@ -534,7 +557,9 @@ Licensed under the MIT License. src/test/resources/circuit-breaker-misc-gateway-testng.xml - + + true + @@ -556,7 +581,9 @@ Licensed under the MIT License. src/test/resources/flaky-multi-master-testng.xml - + + true + @@ -578,7 +605,9 @@ Licensed under the MIT License. src/test/resources/fi-multi-master-testng.xml - + + true + @@ -600,7 +629,9 @@ Licensed under the MIT License. src/test/resources/multi-region-testng.xml - + + true + @@ -623,7 +654,9 @@ Licensed under the MIT License. src/test/resources/examples-testng.xml - + + true + @@ -653,7 +686,9 @@ Licensed under the MIT License. src/test/resources/emulator-testng.xml - + + true + @@ -675,7 +710,9 @@ Licensed under the MIT License. src/test/resources/long-emulator-testng.xml - + + true + @@ -697,7 +734,9 @@ Licensed under the MIT License. src/test/resources/emulator-vnext-testng.xml - + + true + @@ -719,7 +758,9 @@ Licensed under the MIT License. src/test/resources/e2e-testng.xml - + + true + @@ -741,7 +782,9 @@ Licensed under the MIT License. src/test/resources/thinclient-testng.xml - + + true + @@ -763,6 +806,9 @@ Licensed under the MIT License. src/test/resources/fi-thinclient-multi-region-testng.xml + + true + @@ -784,6 +830,9 @@ Licensed under the MIT License. src/test/resources/fi-thinclient-multi-master-testng.xml + + true + diff --git a/sdk/cosmos/azure-cosmos/pom.xml b/sdk/cosmos/azure-cosmos/pom.xml index b474bbe5f73b..cabccf8bde16 100644 --- a/sdk/cosmos/azure-cosmos/pom.xml +++ b/sdk/cosmos/azure-cosmos/pom.xml @@ -146,6 +146,9 @@ Licensed under the MIT License. 3.5.3 unit + + true + %regex[.*] diff --git a/sdk/cosmos/fabric-cosmos-spark-auth_3/pom.xml b/sdk/cosmos/fabric-cosmos-spark-auth_3/pom.xml index 2941fa16041d..080f79dd7b3a 100644 --- a/sdk/cosmos/fabric-cosmos-spark-auth_3/pom.xml +++ b/sdk/cosmos/fabric-cosmos-spark-auth_3/pom.xml @@ -506,6 +506,9 @@ maven-surefire-plugin 3.5.3 + + true + **/*.* **/*Test.* From e84212431f282b5ec05a329e235b0e2ddcd2e12d Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Wed, 12 Nov 2025 00:44:33 +0000 Subject: [PATCH 34/65] Iterating on tests --- sdk/cosmos/azure-cosmos-benchmark/pom.xml | 4 + sdk/cosmos/azure-cosmos-encryption/pom.xml | 4 + .../CosmosNettyLeakDetectorFactory.java | 27 +++++- .../cosmos/encryption/TestSuiteBase.java | 12 --- sdk/cosmos/azure-cosmos-kafka-connect/pom.xml | 4 + .../pom.xml | 10 +++ .../azure-cosmos-spark_3-3_2-12/pom.xml | 10 +++ .../azure-cosmos-spark_3-4_2-12/pom.xml | 10 +++ .../azure-cosmos-spark_3-5_2-12/pom.xml | 10 +++ sdk/cosmos/azure-cosmos-spark_3/pom.xml | 10 +++ sdk/cosmos/azure-cosmos-test/pom.xml | 4 + sdk/cosmos/azure-cosmos-tests/pom.xml | 4 + ...ChangeFeedContinuationTokenUtilsTests.java | 2 +- .../CosmosNettyLeakDetectorFactory.java | 27 +++++- .../azure/cosmos/DistributedClientTest.java | 13 ++- ...ExcludedRegionWithFaultInjectionTests.java | 4 +- .../SessionRetryOptionsTests.java | 6 +- .../rntbd/RntbdRequestRecordTests.java | 64 +++++++------- .../rntbd/RntbdTokenTests.java | 87 +++++++++++-------- .../http/ReactorNettyHttpClientTest.java | 5 +- .../com/azure/cosmos/rx/OfferQueryTest.java | 59 ++++++++----- .../azure/cosmos/rx/ReadFeedOffersTest.java | 58 +++++++------ .../com/azure/cosmos/rx/ReadFeedPkrTests.java | 6 +- .../com/azure/cosmos/rx/TestSuiteBase.java | 12 --- sdk/cosmos/azure-cosmos/pom.xml | 4 + sdk/cosmos/fabric-cosmos-spark-auth_3/pom.xml | 10 +++ 26 files changed, 306 insertions(+), 160 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-benchmark/pom.xml b/sdk/cosmos/azure-cosmos-benchmark/pom.xml index 33c3f11d8292..243197216e95 100644 --- a/sdk/cosmos/azure-cosmos-benchmark/pom.xml +++ b/sdk/cosmos/azure-cosmos-benchmark/pom.xml @@ -214,6 +214,10 @@ Licensed under the MIT License. surefire.testng.verbose 2 + + listener + com.azure.cosmos.CosmosNettyLeakDetectorFactory + true diff --git a/sdk/cosmos/azure-cosmos-encryption/pom.xml b/sdk/cosmos/azure-cosmos-encryption/pom.xml index 1bcea530e743..9444f25232a7 100644 --- a/sdk/cosmos/azure-cosmos-encryption/pom.xml +++ b/sdk/cosmos/azure-cosmos-encryption/pom.xml @@ -228,6 +228,10 @@ Licensed under the MIT License. surefire.testng.verbose 2 + + listener + com.azure.cosmos.encryption.CosmosNettyLeakDetectorFactory + true diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java index 42f42a78102e..99d5a2e8683f 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java @@ -16,6 +16,7 @@ import org.testng.IInvokedMethod; import org.testng.IInvokedMethodListener; import org.testng.ITestClass; +import org.testng.ITestNGMethod; import org.testng.ITestResult; import java.lang.management.BufferPoolMXBean; @@ -70,7 +71,7 @@ public void onBeforeClass(ITestClass testClass) { @Override public void onAfterClass(ITestClass testClass) { - // Unfortunately can't use this in TestNG 7.51 because execution is not symmetric + // Unfortunately can't use this consistently in TestNG 7.51 because execution is not symmetric // IClassListener.onBeforeClass // TestClassBase.@BeforeClass // TestClass.@BeforeClass @@ -81,14 +82,32 @@ public void onAfterClass(ITestClass testClass) { // -Dtestng.listener.execution.symmetric=true allows, but this is only available // in TestNG 7.7.1 - which requires Java11 // So, this class simulates this behavior by hooking into IInvokedMethodListener + // If the test class itself does not have @afterClass we execute the logic here + + ITestNGMethod[] afterClassMethods = testClass.getAfterClassMethods(); + boolean testClassHasAfterClassMethods = afterClassMethods != null && afterClassMethods.length > 0; + if (!testClassHasAfterClassMethods) { + try { + this.onAfterClassCore(testClass); + } catch (Throwable t) { + // decide if you want to fail the suite or just log + System.err.println("Symmetric afterClass failed for " + testClass.getRealClass().getCanonicalName()); + t.printStackTrace(); + } + } } @Override public void afterInvocation(IInvokedMethod method, ITestResult result) { - if (method.isConfigurationMethod() + ITestClass testClass = (ITestClass)result.getTestClass(); + ITestNGMethod[] afterClassMethods = testClass.getAfterClassMethods(); + boolean testClassHasAfterClassMethods = afterClassMethods != null && afterClassMethods.length > 0; + + if (testClassHasAfterClassMethods + && method.isConfigurationMethod() && method.getTestMethod().isAfterClassConfiguration()) { - // <-- This point is guaranteed to be AFTER the class’s @AfterClass ran - ITestClass testClass = (ITestClass)result.getTestClass(); + + // <-- This point is guaranteed to be AFTER the class’s @AfterClass ran if any existed try { this.onAfterClassCore(testClass); } catch (Throwable t) { diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java index aae4f2b707c2..9186c4099d70 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/TestSuiteBase.java @@ -793,18 +793,6 @@ static protected void safeDeleteCollection(CosmosAsyncDatabase database, String } } - static protected void safeCloseAsync(CosmosAsyncClient client) { - if (client != null) { - new Thread(() -> { - try { - client.close(); - } catch (Exception e) { - logger.error("failed to close client", e); - } - }).start(); - } - } - static protected void safeClose(CosmosAsyncClient client) { if (client != null) { try { diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml b/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml index 9b5fc2d64295..3de211365b3e 100644 --- a/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml +++ b/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml @@ -287,6 +287,10 @@ Licensed under the MIT License. surefire.testng.verbose 2 + + listener + com.azure.cosmos.CosmosNettyLeakDetectorFactory + true diff --git a/sdk/cosmos/azure-cosmos-spark-account-data-resolver-sample/pom.xml b/sdk/cosmos/azure-cosmos-spark-account-data-resolver-sample/pom.xml index 5025828b0ae1..88d2e504a2c6 100644 --- a/sdk/cosmos/azure-cosmos-spark-account-data-resolver-sample/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark-account-data-resolver-sample/pom.xml @@ -706,6 +706,16 @@ **/*Spec.* true + + + surefire.testng.verbose + 2 + + + listener + com.azure.cosmos.CosmosNettyLeakDetectorFactory + + diff --git a/sdk/cosmos/azure-cosmos-spark_3-3_2-12/pom.xml b/sdk/cosmos/azure-cosmos-spark_3-3_2-12/pom.xml index 9e4c51da4257..53a15d614595 100644 --- a/sdk/cosmos/azure-cosmos-spark_3-3_2-12/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark_3-3_2-12/pom.xml @@ -128,6 +128,16 @@ **/*Spec.* true + + + surefire.testng.verbose + 2 + + + listener + com.azure.cosmos.CosmosNettyLeakDetectorFactory + + diff --git a/sdk/cosmos/azure-cosmos-spark_3-4_2-12/pom.xml b/sdk/cosmos/azure-cosmos-spark_3-4_2-12/pom.xml index ed6decd27996..b0639b663d22 100644 --- a/sdk/cosmos/azure-cosmos-spark_3-4_2-12/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark_3-4_2-12/pom.xml @@ -128,6 +128,16 @@ **/*Spec.* true + + + surefire.testng.verbose + 2 + + + listener + com.azure.cosmos.CosmosNettyLeakDetectorFactory + + diff --git a/sdk/cosmos/azure-cosmos-spark_3-5_2-12/pom.xml b/sdk/cosmos/azure-cosmos-spark_3-5_2-12/pom.xml index 1026b640d170..ef0c91ea9914 100644 --- a/sdk/cosmos/azure-cosmos-spark_3-5_2-12/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark_3-5_2-12/pom.xml @@ -128,6 +128,16 @@ **/*Spec.* true + + + surefire.testng.verbose + 2 + + + listener + com.azure.cosmos.CosmosNettyLeakDetectorFactory + + diff --git a/sdk/cosmos/azure-cosmos-spark_3/pom.xml b/sdk/cosmos/azure-cosmos-spark_3/pom.xml index b241bba414c7..a58d1d6dd840 100644 --- a/sdk/cosmos/azure-cosmos-spark_3/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark_3/pom.xml @@ -757,6 +757,16 @@ **/*Spec.* true + + + surefire.testng.verbose + 2 + + + listener + com.azure.cosmos.CosmosNettyLeakDetectorFactory + + diff --git a/sdk/cosmos/azure-cosmos-test/pom.xml b/sdk/cosmos/azure-cosmos-test/pom.xml index 38566c8e94a3..f87510f6525e 100644 --- a/sdk/cosmos/azure-cosmos-test/pom.xml +++ b/sdk/cosmos/azure-cosmos-test/pom.xml @@ -163,6 +163,10 @@ Licensed under the MIT License. surefire.testng.verbose 2 + + listener + com.azure.cosmos.CosmosNettyLeakDetectorFactory + true diff --git a/sdk/cosmos/azure-cosmos-tests/pom.xml b/sdk/cosmos/azure-cosmos-tests/pom.xml index 8ae1a907833e..156467069d05 100644 --- a/sdk/cosmos/azure-cosmos-tests/pom.xml +++ b/sdk/cosmos/azure-cosmos-tests/pom.xml @@ -244,6 +244,10 @@ Licensed under the MIT License. surefire.testng.verbose 2 + + listener + com.azure.cosmos.CosmosNettyLeakDetectorFactory + true diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ChangeFeedContinuationTokenUtilsTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ChangeFeedContinuationTokenUtilsTests.java index 2bcb312b0dd1..cc0082e492d9 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ChangeFeedContinuationTokenUtilsTests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ChangeFeedContinuationTokenUtilsTests.java @@ -159,7 +159,7 @@ public void extractContinuationTokens() { @AfterClass(groups = { "emulator" }, timeOut = 3 * SHUTDOWN_TIMEOUT, alwaysRun = true) public void afterClass() { logger.info("starting ...."); - safeCloseAsync(this.client); + safeClose(this.client); } private static class TestItem { diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java index 9f9a6f071889..eccf0760461f 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java @@ -16,6 +16,7 @@ import org.testng.IInvokedMethod; import org.testng.IInvokedMethodListener; import org.testng.ITestClass; +import org.testng.ITestNGMethod; import org.testng.ITestResult; import java.lang.management.BufferPoolMXBean; @@ -70,7 +71,7 @@ public void onBeforeClass(ITestClass testClass) { @Override public void onAfterClass(ITestClass testClass) { - // Unfortunately can't use this in TestNG 7.51 because execution is not symmetric + // Unfortunately can't use this consistently in TestNG 7.51 because execution is not symmetric // IClassListener.onBeforeClass // TestClassBase.@BeforeClass // TestClass.@BeforeClass @@ -81,14 +82,32 @@ public void onAfterClass(ITestClass testClass) { // -Dtestng.listener.execution.symmetric=true allows, but this is only available // in TestNG 7.7.1 - which requires Java11 // So, this class simulates this behavior by hooking into IInvokedMethodListener + // If the test class itself does not have @afterClass we execute the logic here + + ITestNGMethod[] afterClassMethods = testClass.getAfterClassMethods(); + boolean testClassHasAfterClassMethods = afterClassMethods != null && afterClassMethods.length > 0; + if (!testClassHasAfterClassMethods) { + try { + this.onAfterClassCore(testClass); + } catch (Throwable t) { + // decide if you want to fail the suite or just log + System.err.println("Symmetric afterClass failed for " + testClass.getRealClass().getCanonicalName()); + t.printStackTrace(); + } + } } @Override public void afterInvocation(IInvokedMethod method, ITestResult result) { - if (method.isConfigurationMethod() + ITestClass testClass = (ITestClass)result.getTestClass(); + ITestNGMethod[] afterClassMethods = testClass.getAfterClassMethods(); + boolean testClassHasAfterClassMethods = afterClassMethods != null && afterClassMethods.length > 0; + + if (testClassHasAfterClassMethods + && method.isConfigurationMethod() && method.getTestMethod().isAfterClassConfiguration()) { - // <-- This point is guaranteed to be AFTER the class’s @AfterClass ran - ITestClass testClass = (ITestClass)result.getTestClass(); + + // <-- This point is guaranteed to be AFTER the class’s @AfterClass ran if any existed try { this.onAfterClassCore(testClass); } catch (Throwable t) { diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/DistributedClientTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/DistributedClientTest.java index b88bd63db110..156b116dc8b0 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/DistributedClientTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/DistributedClientTest.java @@ -44,13 +44,18 @@ public void createDocument() throws Exception { CosmosClientMetadataCachesSnapshot state = new CosmosClientMetadataCachesSnapshot(); RxClientCollectionCache.serialize(state, cache); - CosmosAsyncClient newClient = new CosmosClientBuilder().endpoint(TestConfigurations.HOST) + try (CosmosAsyncClient newClient = new CosmosClientBuilder().endpoint(TestConfigurations.HOST) .key(TestConfigurations.MASTER_KEY) .metadataCaches(state) - .buildAsyncClient(); + .buildAsyncClient()) { - // TODO: moderakh we should somehow verify that to collection fetch request is made and the existing collection cache is used. - newClient.getDatabase(container.getDatabase().getId()).getContainer(container.getId()).readItem(id, new PartitionKey(id), ObjectNode.class).block(); + // TODO: moderakh we should somehow verify that to collection fetch request is made and the existing collection cache is used. + newClient + .getDatabase(container.getDatabase().getId()) + .getContainer(container.getId()) + .readItem(id, new PartitionKey(id), ObjectNode.class) + .block(); + } } @BeforeClass(groups = { "emulator" }, timeOut = SETUP_TIMEOUT) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/ExcludedRegionWithFaultInjectionTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/ExcludedRegionWithFaultInjectionTests.java index 79bc5818cc92..3b20cd2ec3a5 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/ExcludedRegionWithFaultInjectionTests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/ExcludedRegionWithFaultInjectionTests.java @@ -2422,13 +2422,13 @@ private void execute(MutationTestConfig mutationTestConfig, boolean shouldInject } finally { System.clearProperty("COSMOS.MAX_RETRIES_IN_LOCAL_REGION_WHEN_REMOTE_REGION_PREFERRED"); - safeCloseAsync(clientWithPreferredRegions); + safeClose(clientWithPreferredRegions); } } @AfterClass(groups = {"multi-master"}) public void afterClass() { - safeCloseAsync(this.cosmosAsyncClient); + safeClose(this.cosmosAsyncClient); } private static List buildFaultInjectionRules( diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/SessionRetryOptionsTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/SessionRetryOptionsTests.java index 261ae9ee0a27..c619b5c1d83a 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/SessionRetryOptionsTests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/SessionRetryOptionsTests.java @@ -279,7 +279,7 @@ public void nonWriteOperation_WithReadSessionUnavailable_test( } } finally { System.clearProperty("COSMOS.MAX_RETRIES_IN_LOCAL_REGION_WHEN_REMOTE_REGION_PREFERRED"); - safeCloseAsync(clientWithPreferredRegions); + safeClose(clientWithPreferredRegions); } } @@ -354,13 +354,13 @@ public void writeOperation_withReadSessionUnavailable_test( } } finally { System.clearProperty("COSMOS.MAX_RETRIES_IN_LOCAL_REGION_WHEN_REMOTE_REGION_PREFERRED"); - safeCloseAsync(clientWithPreferredRegions); + safeClose(clientWithPreferredRegions); } } @AfterClass(groups = {"multi-master"}, timeOut = SHUTDOWN_TIMEOUT) public void afterClass() { - safeCloseAsync(cosmosAsyncClient); + safeClose(cosmosAsyncClient); } private Map getRegionMap(DatabaseAccount databaseAccount, boolean writeOnly) { diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdRequestRecordTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdRequestRecordTests.java index 5afe521bcc51..a6f11a5f7c82 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdRequestRecordTests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdRequestRecordTests.java @@ -46,21 +46,24 @@ public void expireRecord(OperationType operationType, boolean requestSent, Class new Uri(new URI("http://localhost/replica-path").toString()) ); - RntbdRequestTimer requestTimer = new RntbdRequestTimer(5000, 5000); - RntbdRequestRecord record = new AsyncRntbdRequestRecord(requestArgs, requestTimer); - if (requestSent) { - record.setSendingRequestHasStarted(); - } - record.expire(); - - try{ - record.get(); - fail("RntbdRequestRecord should complete with exception"); - } catch (ExecutionException e) { - Throwable innerException = e.getCause(); - assertThat(innerException).isInstanceOf(exceptionType); - } catch (Exception e) { - fail("Wrong exception"); + try (RntbdRequestTimer requestTimer = + new RntbdRequestTimer(5000, 5000)) { + + RntbdRequestRecord record = new AsyncRntbdRequestRecord(requestArgs, requestTimer); + if (requestSent) { + record.setSendingRequestHasStarted(); + } + record.expire(); + + try { + record.get(); + fail("RntbdRequestRecord should complete with exception"); + } catch (ExecutionException e) { + Throwable innerException = e.getCause(); + assertThat(innerException).isInstanceOf(exceptionType); + } catch (Exception e) { + fail("Wrong exception"); + } } } @@ -72,22 +75,25 @@ public void cancelRecord() throws URISyntaxException, InterruptedException, Json new Uri(new URI("http://localhost/replica-path").toString()) ); - RntbdRequestTimer requestTimer = new RntbdRequestTimer(5000, 5000); - RntbdRequestRecord record = new AsyncRntbdRequestRecord(requestArgs, requestTimer); - Mono result = Mono.fromFuture(record) - .doOnNext(storeResponse -> fail("Record got cancelled should not reach here")) - .doOnError(throwable -> fail("Record got cancelled should not reach here")); + try (RntbdRequestTimer requestTimer = + new RntbdRequestTimer(5000, 5000)) { + + RntbdRequestRecord record = new AsyncRntbdRequestRecord(requestArgs, requestTimer); + Mono result = Mono.fromFuture(record) + .doOnNext(storeResponse -> fail("Record got cancelled should not reach here")) + .doOnError(throwable -> fail("Record got cancelled should not reach here")); - result.cancelOn(Schedulers.boundedElastic()).subscribe().dispose(); + result.cancelOn(Schedulers.boundedElastic()).subscribe().dispose(); - Thread.sleep(100); - assertThat(record.isCancelled()).isTrue(); + Thread.sleep(100); + assertThat(record.isCancelled()).isTrue(); - String jsonString = record.toString(); - String statusString = "{\"done\":true,\"cancelled\":true,\"completedExceptionally\":true,\"error\":{\"type\":\"java.util.concurrent.CancellationException\"}}"; - JsonNode jsonNode = Utils.getSimpleObjectMapper().readTree(jsonString); - JsonNode errorStatus = jsonNode.get("RntbdRequestRecord").get("status"); - assertThat(errorStatus).isNotNull(); - assertThat(errorStatus.toString()).isEqualTo(statusString); + String jsonString = record.toString(); + String statusString = "{\"done\":true,\"cancelled\":true,\"completedExceptionally\":true,\"error\":{\"type\":\"java.util.concurrent.CancellationException\"}}"; + JsonNode jsonNode = Utils.getSimpleObjectMapper().readTree(jsonString); + JsonNode errorStatus = jsonNode.get("RntbdRequestRecord").get("status"); + assertThat(errorStatus).isNotNull(); + assertThat(errorStatus.toString()).isEqualTo(statusString); + } } } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdTokenTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdTokenTests.java index 6de130598613..f417bec3061b 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdTokenTests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdTokenTests.java @@ -15,50 +15,63 @@ public class RntbdTokenTests { private static final Random rnd = new Random(); @Test(groups = { "unit" }) public void getValueIsIdempotent() { - RntbdToken token = RntbdToken.create(RntbdConstants.RntbdRequestHeader.EffectivePartitionKey); - byte[] blob = new byte[10]; - rnd.nextBytes(blob); - token.setValue(blob); + ByteBuf buffer = null; + ByteBuf byteBufValue1 = null; - String expectedJson = "{\"id\":90,\"name\":\"EffectivePartitionKey\",\"present\":true," - + "\"required\":false,\"value\":\"" - + Base64.getEncoder().encodeToString(blob) - + "\",\"tokenType\":\"Bytes\"}"; - assertThat(RntbdObjectMapper.toJson(token)).isEqualTo(expectedJson); + try { + RntbdToken token = RntbdToken.create(RntbdConstants.RntbdRequestHeader.EffectivePartitionKey); + byte[] blob = new byte[10]; + rnd.nextBytes(blob); + token.setValue(blob); - Object value1 = token.getValue(); - Object value2 = token.getValue(); - assertThat(value1).isSameAs(value2); - assertThat(value1).isSameAs(blob); + String expectedJson = "{\"id\":90,\"name\":\"EffectivePartitionKey\",\"present\":true," + + "\"required\":false,\"value\":\"" + + Base64.getEncoder().encodeToString(blob) + + "\",\"tokenType\":\"Bytes\"}"; + assertThat(RntbdObjectMapper.toJson(token)).isEqualTo(expectedJson); - assertThat(RntbdObjectMapper.toJson(token)).isEqualTo(expectedJson); + Object value1 = token.getValue(); + Object value2 = token.getValue(); + assertThat(value1).isSameAs(value2); + assertThat(value1).isSameAs(blob); - ByteBuf buffer = Unpooled.buffer(1024); - token.encode(buffer); + assertThat(RntbdObjectMapper.toJson(token)).isEqualTo(expectedJson); + buffer = Unpooled.buffer(1024); + token.encode(buffer); - RntbdToken decodedToken = RntbdToken.create(RntbdConstants.RntbdRequestHeader.EffectivePartitionKey); - // skipping 3 bytes (2 bytes for header id + 1 byte for token type) - buffer.readerIndex(3); - // when decoding the RntbdToken.value is a ByteBuffer - not a byte[] - testing this path for idempotency as well - decodedToken.decode(buffer); - assertThat(RntbdObjectMapper.toJson(decodedToken)).isEqualTo(expectedJson); + RntbdToken decodedToken = RntbdToken.create(RntbdConstants.RntbdRequestHeader.EffectivePartitionKey); + // skipping 3 bytes (2 bytes for header id + 1 byte for token type) + buffer.readerIndex(3); + // when decoding the RntbdToken.value is a ByteBuffer - not a byte[] - testing this path for idempotency as well + decodedToken.decode(buffer); + assertThat(RntbdObjectMapper.toJson(decodedToken)).isEqualTo(expectedJson); - value1 = decodedToken.getValue(); - assertThat(value1).isInstanceOf(ByteBuf.class); - ByteBuf byteBufValue1 = (ByteBuf)value1; - assertThat(byteBufValue1.readableBytes()).isEqualTo(10); - byte[] byteArray1 = new byte[10]; - byteBufValue1.getBytes(byteBufValue1.readerIndex(), byteArray1); - assertThat(byteArray1).isEqualTo(blob); + value1 = decodedToken.getValue(); + assertThat(value1).isInstanceOf(ByteBuf.class); - value2 = decodedToken.getValue(); - assertThat(value1).isSameAs(value2); - ByteBuf byteBufValue2 = (ByteBuf)value2; - assertThat(byteBufValue2.readableBytes()).isEqualTo(10); - byte[] byteArray2 = new byte[10]; - byteBufValue2.getBytes(byteBufValue2.readerIndex(), byteArray2); - assertThat(byteArray2).isEqualTo(blob); + byteBufValue1 = (ByteBuf) value1; + assertThat(byteBufValue1.readableBytes()).isEqualTo(10); + byte[] byteArray1 = new byte[10]; + byteBufValue1.getBytes(byteBufValue1.readerIndex(), byteArray1); + assertThat(byteArray1).isEqualTo(blob); - assertThat(RntbdObjectMapper.toJson(decodedToken)).isEqualTo(expectedJson); + value2 = decodedToken.getValue(); + assertThat(value1).isSameAs(value2); + ByteBuf byteBufValue2 = (ByteBuf) value2; + assertThat(byteBufValue2.readableBytes()).isEqualTo(10); + byte[] byteArray2 = new byte[10]; + byteBufValue2.getBytes(byteBufValue2.readerIndex(), byteArray2); + assertThat(byteArray2).isEqualTo(blob); + + assertThat(RntbdObjectMapper.toJson(decodedToken)).isEqualTo(expectedJson); + } finally { + if (buffer != null) { + buffer.release(); + } + + if (byteBufValue1 != null) { + byteBufValue1.release(); + } + } } } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/http/ReactorNettyHttpClientTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/http/ReactorNettyHttpClientTest.java index 74a00185967a..38991fa0f044 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/http/ReactorNettyHttpClientTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/http/ReactorNettyHttpClientTest.java @@ -4,6 +4,8 @@ package com.azure.cosmos.implementation.http; +import com.azure.cosmos.CosmosNettyLeakDetectorFactory; +import com.azure.cosmos.TestNGLogListener; import com.azure.cosmos.implementation.Configs; import com.azure.cosmos.implementation.LifeCycleUtils; import com.azure.cosmos.implementation.directconnectivity.ReflectionUtils; @@ -12,6 +14,7 @@ import org.slf4j.LoggerFactory; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; +import org.testng.annotations.Listeners; import org.testng.annotations.Test; import java.time.Duration; @@ -32,7 +35,7 @@ public void before_ReactorNettyHttpClientTest() { this.reactorNettyHttpClient = HttpClient.createFixed(new HttpClientConfig(new Configs())); } - @AfterClass(groups = "unit") + @AfterClass(groups = "unit", alwaysRun = true) public void after_ReactorNettyHttpClientTest() { LifeCycleUtils.closeQuietly(reactorNettyHttpClient); } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OfferQueryTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OfferQueryTest.java index 20318c661f6c..d52803c0336f 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OfferQueryTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OfferQueryTest.java @@ -63,14 +63,17 @@ public void queryOffersWithFilter() throws Exception { CosmosQueryRequestOptions options = new CosmosQueryRequestOptions(); QueryFeedOperationState dummyState = TestUtils .createDummyQueryFeedOperationState(ResourceType.Offer, OperationType.Query, options, client); + QueryFeedOperationState dummyState2 = null; try { ModelBridgeInternal.setQueryRequestOptionsMaxItemCount(options, 2); Flux> queryObservable = client.queryOffers( query, dummyState); + dummyState2 = TestUtils + .createDummyQueryFeedOperationState(ResourceType.Offer, OperationType.ReadFeed, options, client); List allOffers = client - .readOffers(TestUtils.createDummyQueryFeedOperationState(ResourceType.Offer, OperationType.ReadFeed, options, client)) + .readOffers(dummyState2) .flatMap(f -> Flux.fromIterable(f.getResults())).collectList().single().block(); List expectedOffers = allOffers.stream().filter(o -> collectionResourceId.equals(o.getString("offerResourceId"))).collect(Collectors.toList()); @@ -90,6 +93,7 @@ public void queryOffersWithFilter() throws Exception { validateQuerySuccess(queryObservable, validator, 10000); } finally { safeClose(dummyState); + safeClose(dummyState2); } } @@ -105,13 +109,16 @@ public void queryOffersFilterMorePages() throws Exception { QueryFeedOperationState dummyState = TestUtils .createDummyQueryFeedOperationState(ResourceType.Offer, OperationType.Query, options, client); + QueryFeedOperationState dummyState2 = null; try { Flux> queryObservable = client.queryOffers( query, dummyState); + dummyState2 = TestUtils + .createDummyQueryFeedOperationState(ResourceType.Offer, OperationType.ReadFeed, new CosmosQueryRequestOptions(), client); List expectedOffers = client - .readOffers(TestUtils.createDummyQueryFeedOperationState(ResourceType.Offer, OperationType.ReadFeed, new CosmosQueryRequestOptions(), client)) + .readOffers(dummyState2) .flatMap(f -> Flux.fromIterable(f.getResults())) .collectList() .single().block() @@ -134,6 +141,7 @@ public void queryOffersFilterMorePages() throws Exception { validateQuerySuccess(queryObservable, validator, 10000); } finally { safeClose(dummyState); + safeClose(dummyState2); } } @@ -142,30 +150,35 @@ public void queryCollections_NoResults() throws Exception { String query = "SELECT * from root r where r.id = '2'"; CosmosQueryRequestOptions options = new CosmosQueryRequestOptions(); - CosmosAsyncClient cosmosClient = new CosmosClientBuilder() + try(CosmosAsyncClient cosmosClient = new CosmosClientBuilder() .key(TestConfigurations.MASTER_KEY) .endpoint(TestConfigurations.HOST) - .buildAsyncClient(); - QueryFeedOperationState dummyState = new QueryFeedOperationState( - cosmosClient, - "SomeSpanName", - "SomeDBName", - "SomeContainerName", - ResourceType.Document, - OperationType.Query, - null, - options, - new CosmosPagedFluxOptions() - ); - Flux> queryObservable = client.queryCollections(getDatabaseLink(), query, dummyState); - - FeedResponseListValidator validator = new FeedResponseListValidator.Builder() - .containsExactly(new ArrayList<>()) - .numberOfPages(1) - .pageSatisfy(0, new FeedResponseValidator.Builder() + .buildAsyncClient()) { + QueryFeedOperationState dummyState = new QueryFeedOperationState( + cosmosClient, + "SomeSpanName", + "SomeDBName", + "SomeContainerName", + ResourceType.Document, + OperationType.Query, + null, + options, + new CosmosPagedFluxOptions() + ); + try { + Flux> queryObservable = client.queryCollections(getDatabaseLink(), query, dummyState); + + FeedResponseListValidator validator = new FeedResponseListValidator.Builder() + .containsExactly(new ArrayList<>()) + .numberOfPages(1) + .pageSatisfy(0, new FeedResponseValidator.Builder() .requestChargeGreaterThanOrEqualTo(1.0).build()) - .build(); - validateQuerySuccess(queryObservable, validator); + .build(); + validateQuerySuccess(queryObservable, validator); + } finally { + safeClose(dummyState); + } + } } @BeforeClass(groups = { "query" }, timeOut = SETUP_TIMEOUT) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedOffersTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedOffersTest.java index 1c9627502970..6a2bccf66250 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedOffersTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedOffersTest.java @@ -59,35 +59,41 @@ public void readOffers() throws Exception { CosmosQueryRequestOptions options = new CosmosQueryRequestOptions(); ModelBridgeInternal.setQueryRequestOptionsMaxItemCount(options, 2); - CosmosAsyncClient cosmosClient = new CosmosClientBuilder() + try (CosmosAsyncClient cosmosClient = new CosmosClientBuilder() .key(TestConfigurations.MASTER_KEY) .endpoint(TestConfigurations.HOST) - .buildAsyncClient(); - QueryFeedOperationState dummyState = new QueryFeedOperationState( - cosmosClient, - "SomeSpanName", - "SomeDBName", - "SomeContainerName", - ResourceType.Document, - OperationType.Query, - null, - options, - new CosmosPagedFluxOptions() - ); - - Flux> feedObservable = client.readOffers(dummyState); - - int maxItemCount = ModelBridgeInternal.getMaxItemCountFromQueryRequestOptions(options); - int expectedPageSize = (allOffers.size() + maxItemCount - 1) / maxItemCount; - - FeedResponseListValidator validator = new FeedResponseListValidator.Builder() - .totalSize(allOffers.size()) - .exactlyContainsInAnyOrder(allOffers.stream().map(d -> d.getResourceId()).collect(Collectors.toList())) - .numberOfPages(expectedPageSize) - .pageSatisfy(0, new FeedResponseValidator.Builder() + .buildAsyncClient()) { + QueryFeedOperationState dummyState = new QueryFeedOperationState( + cosmosClient, + "SomeSpanName", + "SomeDBName", + "SomeContainerName", + ResourceType.Document, + OperationType.Query, + null, + options, + new CosmosPagedFluxOptions() + ); + + + try { + Flux> feedObservable = client.readOffers(dummyState); + + int maxItemCount = ModelBridgeInternal.getMaxItemCountFromQueryRequestOptions(options); + int expectedPageSize = (allOffers.size() + maxItemCount - 1) / maxItemCount; + + FeedResponseListValidator validator = new FeedResponseListValidator.Builder() + .totalSize(allOffers.size()) + .exactlyContainsInAnyOrder(allOffers.stream().map(d -> d.getResourceId()).collect(Collectors.toList())) + .numberOfPages(expectedPageSize) + .pageSatisfy(0, new FeedResponseValidator.Builder() .requestChargeGreaterThanOrEqualTo(1.0).build()) - .build(); - validateQuerySuccess(feedObservable, validator, FEED_TIMEOUT); + .build(); + validateQuerySuccess(feedObservable, validator, FEED_TIMEOUT); + } finally { + safeClose(dummyState); + } + } } @BeforeClass(groups = { "query" }, timeOut = SETUP_TIMEOUT) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedPkrTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedPkrTests.java index 16764539b983..680b55050123 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedPkrTests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedPkrTests.java @@ -2,6 +2,7 @@ // Licensed under the MIT License. package com.azure.cosmos.rx; +import com.azure.cosmos.CosmosAsyncClient; import com.azure.cosmos.CosmosAsyncContainer; import com.azure.cosmos.CosmosAsyncDatabase; import com.azure.cosmos.CosmosBridgeInternal; @@ -49,8 +50,9 @@ public void readPartitionKeyRanges() throws Exception { @BeforeClass(groups = { "query" }, timeOut = SETUP_TIMEOUT) public void before_ReadFeedPkrTests() { - client = CosmosBridgeInternal.getAsyncDocumentClient(getClientBuilder().buildAsyncClient()); - createdDatabase = getSharedCosmosDatabase(getClientBuilder().buildAsyncClient()); + CosmosAsyncClient cosmosAsyncClient = getClientBuilder().buildAsyncClient(); + client = CosmosBridgeInternal.getAsyncDocumentClient(cosmosAsyncClient); + createdDatabase = getSharedCosmosDatabase(cosmosAsyncClient); createdCollection = createCollection(createdDatabase, getCollectionDefinition(), new CosmosContainerRequestOptions()); diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java index 6691ef9ae322..e17c95012704 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java @@ -917,18 +917,6 @@ static protected void safeDeleteCollection(CosmosAsyncDatabase database, String } } - static protected void safeCloseAsync(CosmosAsyncClient client) { - if (client != null) { - new Thread(() -> { - try { - client.close(); - } catch (Exception e) { - logger.error("failed to close client", e); - } - }).start(); - } - } - static protected void safeClose(QueryFeedOperationState state) { if (state != null) { safeClose(state.getClient()); diff --git a/sdk/cosmos/azure-cosmos/pom.xml b/sdk/cosmos/azure-cosmos/pom.xml index cabccf8bde16..f28a853bd9f9 100644 --- a/sdk/cosmos/azure-cosmos/pom.xml +++ b/sdk/cosmos/azure-cosmos/pom.xml @@ -157,6 +157,10 @@ Licensed under the MIT License. surefire.testng.verbose 2 + + listener + com.azure.cosmos.CosmosNettyLeakDetectorFactory + true diff --git a/sdk/cosmos/fabric-cosmos-spark-auth_3/pom.xml b/sdk/cosmos/fabric-cosmos-spark-auth_3/pom.xml index 080f79dd7b3a..eb2d31982ffd 100644 --- a/sdk/cosmos/fabric-cosmos-spark-auth_3/pom.xml +++ b/sdk/cosmos/fabric-cosmos-spark-auth_3/pom.xml @@ -516,6 +516,16 @@ **/*Spec.* true + + + surefire.testng.verbose + 2 + + + listener + com.azure.cosmos.CosmosNettyLeakDetectorFactory + + From 1e4bb36331e537674d4ac18d71aa044f6fb4dc7e Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Wed, 12 Nov 2025 01:34:19 +0000 Subject: [PATCH 35/65] Update pom.xml --- sdk/cosmos/azure-cosmos-encryption/pom.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdk/cosmos/azure-cosmos-encryption/pom.xml b/sdk/cosmos/azure-cosmos-encryption/pom.xml index 9444f25232a7..926ce74ff1ce 100644 --- a/sdk/cosmos/azure-cosmos-encryption/pom.xml +++ b/sdk/cosmos/azure-cosmos-encryption/pom.xml @@ -49,6 +49,8 @@ Licensed under the MIT License. --add-opens com.azure.cosmos.encryption/com.azure.cosmos.encryption.util=ALL-UNNAMED --add-opens com.azure.cosmos.encryption/com.azure.cosmos.encryption.models=ALL-UNNAMED --add-opens com.azure.cosmos/com.azure.cosmos.implementation=ALL-UNNAMED + --add-modules java.management + --add-reads com.azure.cosmos.encryption=java.management - true From bb5ba74729b31e951e38356cbcfd5ebfd22fb5ca Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Wed, 12 Nov 2025 02:06:40 +0000 Subject: [PATCH 36/65] Test changes (#47233) --- .../cosmos/benchmark/ReadMyWriteWorkflow.java | 9 + .../implementation/ConsistencyTests2.java | 1 + .../DocumentQuerySpyWireContentTest.java | 24 +- .../RequestHeadersSpyWireTest.java | 4 +- .../cosmos/implementation/SessionTest.java | 223 +++++++++--------- .../DCDocumentCrudTest.java | 6 +- .../directconnectivity/ReflectionUtils.java | 9 + .../RntbdTransportClientTest.java | 5 + .../com/azure/cosmos/rx/OfferQueryTest.java | 84 ++++--- .../azure/cosmos/rx/OfferReadReplaceTest.java | 7 +- .../azure/cosmos/rx/ReadFeedOffersTest.java | 77 +++--- .../azure/cosmos/rx/ResourceTokenTest.java | 8 +- 12 files changed, 234 insertions(+), 223 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-benchmark/src/main/java/com/azure/cosmos/benchmark/ReadMyWriteWorkflow.java b/sdk/cosmos/azure-cosmos-benchmark/src/main/java/com/azure/cosmos/benchmark/ReadMyWriteWorkflow.java index a3d5b79ad4ab..5584f243c419 100644 --- a/sdk/cosmos/azure-cosmos-benchmark/src/main/java/com/azure/cosmos/benchmark/ReadMyWriteWorkflow.java +++ b/sdk/cosmos/azure-cosmos-benchmark/src/main/java/com/azure/cosmos/benchmark/ReadMyWriteWorkflow.java @@ -498,4 +498,13 @@ protected String getDocumentLink(Document doc) { return doc.getSelfLink(); } } + + @Override + void shutdown() { + if (this.client != null) { + this.client.close(); + } + + super.shutdown(); + } } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConsistencyTests2.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConsistencyTests2.java index 38d8c941efa7..1143dfcba0e8 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConsistencyTests2.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConsistencyTests2.java @@ -233,6 +233,7 @@ public void validateNoChargeOnFailedSessionRead() throws Exception { new CosmosClientTelemetryConfig() .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) .build(); + QueryFeedOperationState dummyState = null; try { // CREATE collection diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/DocumentQuerySpyWireContentTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/DocumentQuerySpyWireContentTest.java index fcbee5b7f3e9..5540662212d8 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/DocumentQuerySpyWireContentTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/DocumentQuerySpyWireContentTest.java @@ -95,8 +95,7 @@ public void queryWithContinuationTokenLimit(CosmosQueryRequestOptions options, S client.clearCapturedRequests(); - QueryFeedOperationState dummyState = TestUtils - .createDummyQueryFeedOperationState(ResourceType.Document, OperationType.Query, options, client); + QueryFeedOperationState dummyState = TestUtils.createDummyQueryFeedOperationState(ResourceType.Document, OperationType.Query, options, client); try { Flux> queryObservable = client .queryDocuments( @@ -148,6 +147,11 @@ public Document createDocument(AsyncDocumentClient client, String collectionLink @BeforeClass(groups = { "fast" }, timeOut = SETUP_TIMEOUT) public void before_DocumentQuerySpyWireContentTest() throws Exception { + SpyClientUnderTestFactory.ClientUnderTest oldSnapshot = client; + if (oldSnapshot != null) { + oldSnapshot.close(); + } + client = new SpyClientBuilder(this.clientBuilder()).build(); createdDatabase = SHARED_DATABASE; @@ -177,18 +181,14 @@ public void before_DocumentQuerySpyWireContentTest() throws Exception { options, client ); - try { - // do the query once to ensure the collection is cached. - client.queryDocuments(getMultiPartitionCollectionLink(), "select * from root", state, Document.class) - .then().block(); + // do the query once to ensure the collection is cached. + client.queryDocuments(getMultiPartitionCollectionLink(), "select * from root", state, Document.class) + .then().block(); - // do the query once to ensure the collection is cached. - client.queryDocuments(getSinglePartitionCollectionLink(), "select * from root", state, Document.class) - .then().block(); - } finally { - safeClose(state); - } + // do the query once to ensure the collection is cached. + client.queryDocuments(getSinglePartitionCollectionLink(), "select * from root", state, Document.class) + .then().block(); } @AfterClass(groups = { "fast" }, timeOut = SHUTDOWN_TIMEOUT, alwaysRun = true) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/RequestHeadersSpyWireTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/RequestHeadersSpyWireTest.java index b9cbde1fc7fd..9f5e783d1f67 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/RequestHeadersSpyWireTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/RequestHeadersSpyWireTest.java @@ -137,8 +137,7 @@ public void queryWithMaxIntegratedCacheStaleness(CosmosQueryRequestOptions optio client.clearCapturedRequests(); - QueryFeedOperationState dummyState = TestUtils - .createDummyQueryFeedOperationState(ResourceType.Document, OperationType.Query, options, client); + QueryFeedOperationState dummyState = TestUtils.createDummyQueryFeedOperationState(ResourceType.Document, OperationType.Query, options, client); try { client.queryDocuments( collectionLink, @@ -175,7 +174,6 @@ public void queryWithMaxIntegratedCacheStalenessInNanoseconds() { ); try { - assertThatThrownBy(() -> client .queryDocuments(collectionLink, query, state, Document.class) .blockLast()) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/SessionTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/SessionTest.java index eb1ff945e297..ee03e5e34414 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/SessionTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/SessionTest.java @@ -203,132 +203,129 @@ public void partitionedSessionToken(boolean isNameBased) throws NoSuchMethodExce spyClient ); - try { + spyClient.queryDocuments(getCollectionLink(isNameBased), query, dummyState, Document.class).blockFirst(); + assertThat(getSessionTokensInRequests()).hasSize(1); + assertThat(getSessionTokensInRequests().get(0)).isNotEmpty(); + assertThat(getSessionTokensInRequests().get(0)).doesNotContain(","); // making sure we have only one scope session token - spyClient.queryDocuments(getCollectionLink(isNameBased), query, dummyState, Document.class).blockFirst(); - assertThat(getSessionTokensInRequests()).hasSize(1); - assertThat(getSessionTokensInRequests().get(0)).isNotEmpty(); - assertThat(getSessionTokensInRequests().get(0)).doesNotContain(","); // making sure we have only one scope session token + // Session token validation for cross partition query + spyClient.clearCapturedRequests(); + queryRequestOptions = new CosmosQueryRequestOptions(); - // Session token validation for cross partition query - spyClient.clearCapturedRequests(); - queryRequestOptions = new CosmosQueryRequestOptions(); - safeClose(dummyState); - dummyState = TestUtils.createDummyQueryFeedOperationState( - ResourceType.Document, - OperationType.Query, - queryRequestOptions, - spyClient - ); - spyClient.queryDocuments(getCollectionLink(isNameBased), query, dummyState, Document.class).blockFirst(); - assertThat(getSessionTokensInRequests().size()).isGreaterThanOrEqualTo(1); - assertThat(getSessionTokensInRequests().get(0)).isNotEmpty(); - assertThat(getSessionTokensInRequests().get(0)).doesNotContain(","); // making sure we have only one scope session token + safeClose(dummyState); + dummyState = TestUtils.createDummyQueryFeedOperationState( + ResourceType.Document, + OperationType.Query, + queryRequestOptions, + spyClient + ); + spyClient.queryDocuments(getCollectionLink(isNameBased), query, dummyState, Document.class).blockFirst(); + assertThat(getSessionTokensInRequests().size()).isGreaterThanOrEqualTo(1); + assertThat(getSessionTokensInRequests().get(0)).isNotEmpty(); + assertThat(getSessionTokensInRequests().get(0)).doesNotContain(","); // making sure we have only one scope session token - // Session token validation for feed ranges query - spyClient.clearCapturedRequests(); - List feedRanges = spyClient.getFeedRanges(getCollectionLink(isNameBased), true).block(); - queryRequestOptions = new CosmosQueryRequestOptions(); - queryRequestOptions.setFeedRange(feedRanges.get(0)); - safeClose(dummyState); - dummyState = TestUtils.createDummyQueryFeedOperationState( - ResourceType.Document, - OperationType.Query, - queryRequestOptions, - spyClient - ); - spyClient.queryDocuments(getCollectionLink(isNameBased), query, dummyState, Document.class).blockFirst(); - assertThat(getSessionTokensInRequests().size()).isGreaterThanOrEqualTo(1); - assertThat(getSessionTokensInRequests().get(0)).isNotEmpty(); - assertThat(getSessionTokensInRequests().get(0)).doesNotContain(","); // making sure we have only one scope session token + // Session token validation for feed ranges query + spyClient.clearCapturedRequests(); + List feedRanges = spyClient.getFeedRanges(getCollectionLink(isNameBased), true).block(); + queryRequestOptions = new CosmosQueryRequestOptions(); + queryRequestOptions.setFeedRange(feedRanges.get(0)); + safeClose(dummyState); + dummyState = TestUtils.createDummyQueryFeedOperationState( + ResourceType.Document, + OperationType.Query, + queryRequestOptions, + spyClient + ); + spyClient.queryDocuments(getCollectionLink(isNameBased), query, dummyState, Document.class).blockFirst(); + assertThat(getSessionTokensInRequests().size()).isGreaterThanOrEqualTo(1); + assertThat(getSessionTokensInRequests().get(0)).isNotEmpty(); + assertThat(getSessionTokensInRequests().get(0)).doesNotContain(","); // making sure we have only one scope session token - // Session token validation for readAll with partition query - spyClient.clearCapturedRequests(); - queryRequestOptions = new CosmosQueryRequestOptions(); - safeClose(dummyState); - dummyState = TestUtils.createDummyQueryFeedOperationState( - ResourceType.Document, - OperationType.ReadFeed, - queryRequestOptions, - spyClient - ); - spyClient.readAllDocuments( - getCollectionLink(isNameBased), - new PartitionKey(documentCreated.getId()), - dummyState, - Document.class).blockFirst(); - assertThat(getSessionTokensInRequests().size()).isEqualTo(1); - assertThat(getSessionTokensInRequests().get(0)).isNotEmpty(); - assertThat(getSessionTokensInRequests().get(0)).doesNotContain(","); // making sure we have only one scope session token + // Session token validation for readAll with partition query + spyClient.clearCapturedRequests(); + queryRequestOptions = new CosmosQueryRequestOptions(); + safeClose(dummyState); + dummyState = TestUtils.createDummyQueryFeedOperationState( + ResourceType.Document, + OperationType.ReadFeed, + queryRequestOptions, + spyClient + ); + spyClient.readAllDocuments( + getCollectionLink(isNameBased), + new PartitionKey(documentCreated.getId()), + dummyState, + Document.class).blockFirst(); + assertThat(getSessionTokensInRequests().size()).isEqualTo(1); + assertThat(getSessionTokensInRequests().get(0)).isNotEmpty(); + assertThat(getSessionTokensInRequests().get(0)).doesNotContain(","); // making sure we have only one scope session token - // Session token validation for readAll with cross partition - spyClient.clearCapturedRequests(); - queryRequestOptions = new CosmosQueryRequestOptions(); + // Session token validation for readAll with cross partition + spyClient.clearCapturedRequests(); + queryRequestOptions = new CosmosQueryRequestOptions(); - safeClose(dummyState); - dummyState = TestUtils.createDummyQueryFeedOperationState( - ResourceType.Document, - OperationType.ReadFeed, - queryRequestOptions, - spyClient - ); + safeClose(dummyState); + dummyState = TestUtils.createDummyQueryFeedOperationState( + ResourceType.Document, + OperationType.ReadFeed, + queryRequestOptions, + spyClient + ); - spyClient.readDocuments(getCollectionLink(isNameBased), dummyState, Document.class).blockFirst(); - assertThat(getSessionTokensInRequests().size()).isGreaterThanOrEqualTo(1); - assertThat(getSessionTokensInRequests().get(0)).isNotEmpty(); - assertThat(getSessionTokensInRequests().get(0)).doesNotContain(","); // making sure we have only one scope session token + spyClient.readDocuments(getCollectionLink(isNameBased), dummyState, Document.class).blockFirst(); + assertThat(getSessionTokensInRequests().size()).isGreaterThanOrEqualTo(1); + assertThat(getSessionTokensInRequests().get(0)).isNotEmpty(); + assertThat(getSessionTokensInRequests().get(0)).doesNotContain(","); // making sure we have only one scope session token - // Session token validation for readMany with cross partition + // Session token validation for readMany with cross partition + spyClient.clearCapturedRequests(); + queryRequestOptions = new CosmosQueryRequestOptions(); + CosmosItemIdentity cosmosItemIdentity = new CosmosItemIdentity(new PartitionKey(documentCreated.getId()), documentCreated.getId()); + List cosmosItemIdentities = new ArrayList<>(); + cosmosItemIdentities.add(cosmosItemIdentity); + safeClose(dummyState); + dummyState = TestUtils.createDummyQueryFeedOperationState(ResourceType.Document, OperationType.Query, queryRequestOptions, spyClient); + spyClient.readMany( + cosmosItemIdentities, + getCollectionLink(isNameBased), + dummyState, + InternalObjectNode.class).block(); + assertThat(getSessionTokensInRequests().size()).isEqualTo(1); + assertThat(getSessionTokensInRequests().get(0)).isNotEmpty(); + assertThat(getSessionTokensInRequests().get(0)).doesNotContain(","); // making sure we have only one scope session token + // session token + + // Session token validation for create in Batch + if(isNameBased) { // Batch only work with name based url spyClient.clearCapturedRequests(); - queryRequestOptions = new CosmosQueryRequestOptions(); - CosmosItemIdentity cosmosItemIdentity = new CosmosItemIdentity(new PartitionKey(documentCreated.getId()), documentCreated.getId()); - List cosmosItemIdentities = new ArrayList<>(); - cosmosItemIdentities.add(cosmosItemIdentity); - safeClose(dummyState); - dummyState = TestUtils - .createDummyQueryFeedOperationState(ResourceType.Document, OperationType.Query, queryRequestOptions, spyClient); - spyClient.readMany( - cosmosItemIdentities, - getCollectionLink(isNameBased), - dummyState, - InternalObjectNode.class).block(); + Document document = newDocument(); + document.set("mypk", document.getId()); + ItemBatchOperation itemBatchOperation = new ItemBatchOperation(CosmosItemOperationType.CREATE, + documentCreated.getId(), new PartitionKey(documentCreated.getId()), new RequestOptions(), document); + List> itemBatchOperations = new ArrayList<>(); + itemBatchOperations.add(itemBatchOperation); + + Method method = SinglePartitionKeyServerBatchRequest.class.getDeclaredMethod("createBatchRequest", + PartitionKey.class, + List.class); + method.setAccessible(true); + SinglePartitionKeyServerBatchRequest serverBatchRequest = + (SinglePartitionKeyServerBatchRequest) method.invoke(SinglePartitionKeyServerBatchRequest.class, new PartitionKey(document.getId()), + itemBatchOperations); + spyClient + .executeBatchRequest( + getCollectionLink(isNameBased), + serverBatchRequest, + new RequestOptions(), + false, + true) + .block(); assertThat(getSessionTokensInRequests().size()).isEqualTo(1); assertThat(getSessionTokensInRequests().get(0)).isNotEmpty(); assertThat(getSessionTokensInRequests().get(0)).doesNotContain(","); // making sure we have only one scope session token - // session token - - // Session token validation for create in Batch - if (isNameBased) { // Batch only work with name based url - spyClient.clearCapturedRequests(); - Document document = newDocument(); - document.set("mypk", document.getId()); - ItemBatchOperation itemBatchOperation = new ItemBatchOperation(CosmosItemOperationType.CREATE, - documentCreated.getId(), new PartitionKey(documentCreated.getId()), new RequestOptions(), document); - List> itemBatchOperations = new ArrayList<>(); - itemBatchOperations.add(itemBatchOperation); - - Method method = SinglePartitionKeyServerBatchRequest.class.getDeclaredMethod("createBatchRequest", - PartitionKey.class, - List.class); - method.setAccessible(true); - SinglePartitionKeyServerBatchRequest serverBatchRequest = - (SinglePartitionKeyServerBatchRequest) method.invoke(SinglePartitionKeyServerBatchRequest.class, new PartitionKey(document.getId()), - itemBatchOperations); - spyClient - .executeBatchRequest( - getCollectionLink(isNameBased), - serverBatchRequest, - new RequestOptions(), - false, - true) - .block(); - assertThat(getSessionTokensInRequests().size()).isEqualTo(1); - assertThat(getSessionTokensInRequests().get(0)).isNotEmpty(); - assertThat(getSessionTokensInRequests().get(0)).doesNotContain(","); // making sure we have only one scope session token - } - } finally { - safeClose(dummyState); } + + safeClose(dummyState); } @Test(groups = { "fast" }, timeOut = TIMEOUT, dataProvider = "sessionTestArgProvider") diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/DCDocumentCrudTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/DCDocumentCrudTest.java index 27969a39fd7c..7e7410836eb5 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/DCDocumentCrudTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/DCDocumentCrudTest.java @@ -229,9 +229,8 @@ public void crossPartitionQuery() { options.setMaxDegreeOfParallelism(-1); ModelBridgeInternal.setQueryRequestOptionsMaxItemCount(options, 100); - QueryFeedOperationState dummyState = TestUtils - .createDummyQueryFeedOperationState(ResourceType.Document, OperationType.Query, options, client); - + QueryFeedOperationState dummyState = + TestUtils.createDummyQueryFeedOperationState(ResourceType.Document, OperationType.Query, options, client); try { Flux> results = client.queryDocuments( getCollectionLink(), @@ -245,6 +244,7 @@ public void crossPartitionQuery() { validateQuerySuccess(results, validator, QUERY_TIMEOUT); validateNoDocumentQueryOperationThroughGateway(); + // validates only the first query for fetching query plan goes to gateway. assertThat(client.getCapturedRequests().stream().filter(r -> r.getResourceType() == ResourceType.Document)).hasSize(1); } finally { diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/ReflectionUtils.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/ReflectionUtils.java index cf126f74b3fe..cf1110aec87f 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/ReflectionUtils.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/ReflectionUtils.java @@ -317,6 +317,15 @@ public static void setTransportClient(StoreReader storeReader, TransportClient t set(storeReader, transportClient, "transportClient"); } + public static void setTransportClient(CosmosClient client, TransportClient transportClient) { + StoreClient storeClient = getStoreClient((RxDocumentClientImpl) CosmosBridgeInternal.getAsyncDocumentClient(client)); + set(storeClient, transportClient, "transportClient"); + ReplicatedResourceClient replicatedResClient = getReplicatedResourceClient(storeClient); + ConsistencyWriter writer = getConsistencyWriter(replicatedResClient); + set(replicatedResClient, transportClient, "transportClient"); + set(writer, transportClient, "transportClient"); + } + public static TransportClient getTransportClient(ReplicatedResourceClient replicatedResourceClient) { return get(TransportClient.class, replicatedResourceClient, "transportClient"); } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/RntbdTransportClientTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/RntbdTransportClientTest.java index 07af54b8c73c..b1c93b2ee7a3 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/RntbdTransportClientTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/RntbdTransportClientTest.java @@ -1138,6 +1138,11 @@ public URI serviceEndpoint() { return null; } + @Override + public URI serverKeyUsedAsActualRemoteAddress() { + return this.remoteURI; + } + @Override public void injectConnectionErrors(String ruleId, double threshold, Class eventType) { throw new NotImplementedException("injectConnectionErrors is not supported in FakeEndpoint"); diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OfferQueryTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OfferQueryTest.java index d52803c0336f..fb814862e6ea 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OfferQueryTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OfferQueryTest.java @@ -61,19 +61,20 @@ public void queryOffersWithFilter() throws Exception { String query = String.format("SELECT * from c where c.offerResourceId = '%s'", collectionResourceId); CosmosQueryRequestOptions options = new CosmosQueryRequestOptions(); - QueryFeedOperationState dummyState = TestUtils + ModelBridgeInternal.setQueryRequestOptionsMaxItemCount(options, 2); + + QueryFeedOperationState queryDummyState = TestUtils .createDummyQueryFeedOperationState(ResourceType.Offer, OperationType.Query, options, client); - QueryFeedOperationState dummyState2 = null; + QueryFeedOperationState offerDummyState = TestUtils + .createDummyQueryFeedOperationState(ResourceType.Offer, OperationType.ReadFeed, options, client); + try { - ModelBridgeInternal.setQueryRequestOptionsMaxItemCount(options, 2); Flux> queryObservable = client.queryOffers( query, - dummyState); + queryDummyState); - dummyState2 = TestUtils - .createDummyQueryFeedOperationState(ResourceType.Offer, OperationType.ReadFeed, options, client); List allOffers = client - .readOffers(dummyState2) + .readOffers(offerDummyState) .flatMap(f -> Flux.fromIterable(f.getResults())).collectList().single().block(); List expectedOffers = allOffers.stream().filter(o -> collectionResourceId.equals(o.getString("offerResourceId"))).collect(Collectors.toList()); @@ -92,8 +93,8 @@ public void queryOffersWithFilter() throws Exception { validateQuerySuccess(queryObservable, validator, 10000); } finally { - safeClose(dummyState); - safeClose(dummyState2); + safeClose(queryDummyState); + safeClose(offerDummyState); } } @@ -107,18 +108,20 @@ public void queryOffersFilterMorePages() throws Exception { CosmosQueryRequestOptions options = new CosmosQueryRequestOptions(); ModelBridgeInternal.setQueryRequestOptionsMaxItemCount(options, 1); - QueryFeedOperationState dummyState = TestUtils + QueryFeedOperationState queryDummyState = TestUtils .createDummyQueryFeedOperationState(ResourceType.Offer, OperationType.Query, options, client); - QueryFeedOperationState dummyState2 = null; + + QueryFeedOperationState offerDummyState = TestUtils + .createDummyQueryFeedOperationState(ResourceType.Offer, OperationType.ReadFeed, new CosmosQueryRequestOptions(), client); + try { + Flux> queryObservable = client.queryOffers( query, - dummyState); + queryDummyState); - dummyState2 = TestUtils - .createDummyQueryFeedOperationState(ResourceType.Offer, OperationType.ReadFeed, new CosmosQueryRequestOptions(), client); List expectedOffers = client - .readOffers(dummyState2) + .readOffers(offerDummyState) .flatMap(f -> Flux.fromIterable(f.getResults())) .collectList() .single().block() @@ -140,8 +143,8 @@ public void queryOffersFilterMorePages() throws Exception { validateQuerySuccess(queryObservable, validator, 10000); } finally { - safeClose(dummyState); - safeClose(dummyState2); + safeClose(queryDummyState); + safeClose(offerDummyState); } } @@ -150,35 +153,30 @@ public void queryCollections_NoResults() throws Exception { String query = "SELECT * from root r where r.id = '2'"; CosmosQueryRequestOptions options = new CosmosQueryRequestOptions(); - try(CosmosAsyncClient cosmosClient = new CosmosClientBuilder() + CosmosAsyncClient cosmosClient = new CosmosClientBuilder() .key(TestConfigurations.MASTER_KEY) .endpoint(TestConfigurations.HOST) - .buildAsyncClient()) { - QueryFeedOperationState dummyState = new QueryFeedOperationState( - cosmosClient, - "SomeSpanName", - "SomeDBName", - "SomeContainerName", - ResourceType.Document, - OperationType.Query, - null, - options, - new CosmosPagedFluxOptions() - ); - try { - Flux> queryObservable = client.queryCollections(getDatabaseLink(), query, dummyState); - - FeedResponseListValidator validator = new FeedResponseListValidator.Builder() - .containsExactly(new ArrayList<>()) - .numberOfPages(1) - .pageSatisfy(0, new FeedResponseValidator.Builder() + .buildAsyncClient(); + QueryFeedOperationState dummyState = new QueryFeedOperationState( + cosmosClient, + "SomeSpanName", + "SomeDBName", + "SomeContainerName", + ResourceType.Document, + OperationType.Query, + null, + options, + new CosmosPagedFluxOptions() + ); + Flux> queryObservable = client.queryCollections(getDatabaseLink(), query, dummyState); + + FeedResponseListValidator validator = new FeedResponseListValidator.Builder() + .containsExactly(new ArrayList<>()) + .numberOfPages(1) + .pageSatisfy(0, new FeedResponseValidator.Builder() .requestChargeGreaterThanOrEqualTo(1.0).build()) - .build(); - validateQuerySuccess(queryObservable, validator); - } finally { - safeClose(dummyState); - } - } + .build(); + validateQuerySuccess(queryObservable, validator); } @BeforeClass(groups = { "query" }, timeOut = SETUP_TIMEOUT) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OfferReadReplaceTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OfferReadReplaceTest.java index c3ac8eb1e5a4..ed4f1a523e71 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OfferReadReplaceTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OfferReadReplaceTest.java @@ -43,14 +43,15 @@ public OfferReadReplaceTest(AsyncDocumentClient.Builder clientBuilder) { @Test(groups = { "emulator" }, timeOut = TIMEOUT) public void readAndReplaceOffer() { - QueryFeedOperationState dummyState = TestUtils.createDummyQueryFeedOperationState( + QueryFeedOperationState offerDummyState = TestUtils.createDummyQueryFeedOperationState( ResourceType.Offer, OperationType.ReadFeed, new CosmosQueryRequestOptions(), client); + try { List offers = client - .readOffers(dummyState) + .readOffers(offerDummyState) .map(FeedResponse::getResults) .flatMap(list -> Flux.fromIterable(list)).collectList().block(); @@ -87,7 +88,7 @@ public void readAndReplaceOffer() { validateSuccess(replaceObservable, validatorForReplace); } finally { - safeClose(dummyState); + safeClose(offerDummyState); } } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedOffersTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedOffersTest.java index 6a2bccf66250..6bc22bd03a2d 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedOffersTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedOffersTest.java @@ -59,41 +59,35 @@ public void readOffers() throws Exception { CosmosQueryRequestOptions options = new CosmosQueryRequestOptions(); ModelBridgeInternal.setQueryRequestOptionsMaxItemCount(options, 2); - try (CosmosAsyncClient cosmosClient = new CosmosClientBuilder() + CosmosAsyncClient cosmosClient = new CosmosClientBuilder() .key(TestConfigurations.MASTER_KEY) .endpoint(TestConfigurations.HOST) - .buildAsyncClient()) { - QueryFeedOperationState dummyState = new QueryFeedOperationState( - cosmosClient, - "SomeSpanName", - "SomeDBName", - "SomeContainerName", - ResourceType.Document, - OperationType.Query, - null, - options, - new CosmosPagedFluxOptions() - ); - - - try { - Flux> feedObservable = client.readOffers(dummyState); - - int maxItemCount = ModelBridgeInternal.getMaxItemCountFromQueryRequestOptions(options); - int expectedPageSize = (allOffers.size() + maxItemCount - 1) / maxItemCount; - - FeedResponseListValidator validator = new FeedResponseListValidator.Builder() - .totalSize(allOffers.size()) - .exactlyContainsInAnyOrder(allOffers.stream().map(d -> d.getResourceId()).collect(Collectors.toList())) - .numberOfPages(expectedPageSize) - .pageSatisfy(0, new FeedResponseValidator.Builder() + .buildAsyncClient(); + QueryFeedOperationState dummyState = new QueryFeedOperationState( + cosmosClient, + "SomeSpanName", + "SomeDBName", + "SomeContainerName", + ResourceType.Document, + OperationType.Query, + null, + options, + new CosmosPagedFluxOptions() + ); + + Flux> feedObservable = client.readOffers(dummyState); + + int maxItemCount = ModelBridgeInternal.getMaxItemCountFromQueryRequestOptions(options); + int expectedPageSize = (allOffers.size() + maxItemCount - 1) / maxItemCount; + + FeedResponseListValidator validator = new FeedResponseListValidator.Builder() + .totalSize(allOffers.size()) + .exactlyContainsInAnyOrder(allOffers.stream().map(d -> d.getResourceId()).collect(Collectors.toList())) + .numberOfPages(expectedPageSize) + .pageSatisfy(0, new FeedResponseValidator.Builder() .requestChargeGreaterThanOrEqualTo(1.0).build()) - .build(); - validateQuerySuccess(feedObservable, validator, FEED_TIMEOUT); - } finally { - safeClose(dummyState); - } - } + .build(); + validateQuerySuccess(feedObservable, validator, FEED_TIMEOUT); } @BeforeClass(groups = { "query" }, timeOut = SETUP_TIMEOUT) @@ -105,20 +99,23 @@ public void before_ReadFeedOffersTest() { createCollections(client); } - QueryFeedOperationState dummyState = TestUtils.createDummyQueryFeedOperationState( + QueryFeedOperationState offerDummyState = TestUtils.createDummyQueryFeedOperationState( ResourceType.Offer, OperationType.ReadFeed, new CosmosQueryRequestOptions(), client ); - allOffers = client.readOffers(dummyState) - .map(FeedResponse::getResults) - .collectList() - .map(list -> list.stream().flatMap(Collection::stream).collect(Collectors.toList())) - .single() - .doFinally(signal -> safeClose(dummyState)) - .block(); + try { + allOffers = client.readOffers(offerDummyState) + .map(FeedResponse::getResults) + .collectList() + .map(list -> list.stream().flatMap(Collection::stream).collect(Collectors.toList())) + .single() + .block(); + } finally { + safeClose(offerDummyState); + } } @AfterClass(groups = { "query" }, timeOut = SHUTDOWN_TIMEOUT, alwaysRun = true) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ResourceTokenTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ResourceTokenTest.java index d42367a5ef49..ead3cdc9f785 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ResourceTokenTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ResourceTokenTest.java @@ -483,11 +483,7 @@ public void queryItemFromResourceToken(DocumentCollection documentCollection, Pe queryRequestOptions.setPartitionKey(partitionKey); dummyState = TestUtils - .createDummyQueryFeedOperationState( - ResourceType.Document, - OperationType.Query, - queryRequestOptions, - asyncClientResourceToken); + .createDummyQueryFeedOperationState(ResourceType.Document, OperationType.Query, queryRequestOptions, asyncClientResourceToken); Flux> queryObservable = asyncClientResourceToken.queryDocuments( @@ -503,8 +499,8 @@ public void queryItemFromResourceToken(DocumentCollection documentCollection, Pe validateQuerySuccess(queryObservable, validator); } finally { - safeClose(dummyState); safeClose(asyncClientResourceToken); + safeClose(dummyState); } } From 8835846225c226589dec393b413e5ee07856ef6b Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Wed, 12 Nov 2025 02:12:49 +0000 Subject: [PATCH 37/65] Update RntbdTransportClientTest.java --- .../directconnectivity/RntbdTransportClientTest.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/RntbdTransportClientTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/RntbdTransportClientTest.java index b1c93b2ee7a3..07af54b8c73c 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/RntbdTransportClientTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/RntbdTransportClientTest.java @@ -1138,11 +1138,6 @@ public URI serviceEndpoint() { return null; } - @Override - public URI serverKeyUsedAsActualRemoteAddress() { - return this.remoteURI; - } - @Override public void injectConnectionErrors(String ruleId, double threshold, Class eventType) { throw new NotImplementedException("injectConnectionErrors is not supported in FakeEndpoint"); From 7fcacf27a4ee745dac1c2ed3540836d827e62605 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Wed, 12 Nov 2025 11:18:10 +0000 Subject: [PATCH 38/65] Updating netty leak detection system properties --- sdk/cosmos/azure-cosmos-benchmark/pom.xml | 42 +++++++++++++ sdk/cosmos/azure-cosmos-encryption/pom.xml | 9 +++ .../CosmosNettyLeakDetectorFactory.java | 7 ++- sdk/cosmos/azure-cosmos-kafka-connect/pom.xml | 15 +++++ .../pom.xml | 3 + .../azure-cosmos-spark_3-3_2-12/pom.xml | 3 + .../azure-cosmos-spark_3-4_2-12/pom.xml | 3 + .../azure-cosmos-spark_3-5_2-12/pom.xml | 3 + sdk/cosmos/azure-cosmos-spark_3/pom.xml | 3 + sdk/cosmos/azure-cosmos-test/pom.xml | 3 + sdk/cosmos/azure-cosmos-tests/pom.xml | 60 +++++++++++++++++++ .../CosmosNettyLeakDetectorFactory.java | 5 +- sdk/cosmos/azure-cosmos/pom.xml | 3 + .../implementation/RxDocumentClientImpl.java | 3 + sdk/cosmos/fabric-cosmos-spark-auth_3/pom.xml | 3 + 15 files changed, 160 insertions(+), 5 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-benchmark/pom.xml b/sdk/cosmos/azure-cosmos-benchmark/pom.xml index 243197216e95..5c9138a92de4 100644 --- a/sdk/cosmos/azure-cosmos-benchmark/pom.xml +++ b/sdk/cosmos/azure-cosmos-benchmark/pom.xml @@ -205,6 +205,9 @@ Licensed under the MIT License. unit true + 1 + 256 + paranoid %regex[.*] @@ -317,6 +320,9 @@ Licensed under the MIT License. true + 1 + 256 + paranoid @@ -341,6 +347,9 @@ Licensed under the MIT License. true + 1 + 256 + paranoid @@ -365,6 +374,9 @@ Licensed under the MIT License. true + 1 + 256 + paranoid @@ -389,6 +401,9 @@ Licensed under the MIT License. true + 1 + 256 + paranoid @@ -413,6 +428,9 @@ Licensed under the MIT License. true + 1 + 256 + paranoid @@ -437,6 +455,9 @@ Licensed under the MIT License. true + 1 + 256 + paranoid @@ -461,6 +482,9 @@ Licensed under the MIT License. true + 1 + 256 + paranoid @@ -485,6 +509,9 @@ Licensed under the MIT License. true + 1 + 256 + paranoid @@ -509,6 +536,9 @@ Licensed under the MIT License. true + 1 + 256 + paranoid @@ -533,6 +563,9 @@ Licensed under the MIT License. true + 1 + 256 + paranoid @@ -558,6 +591,9 @@ Licensed under the MIT License. true + 1 + 256 + paranoid @@ -590,6 +626,9 @@ Licensed under the MIT License. true + 1 + 256 + paranoid @@ -614,6 +653,9 @@ Licensed under the MIT License. true + 1 + 256 + paranoid diff --git a/sdk/cosmos/azure-cosmos-encryption/pom.xml b/sdk/cosmos/azure-cosmos-encryption/pom.xml index 926ce74ff1ce..e4810e35139d 100644 --- a/sdk/cosmos/azure-cosmos-encryption/pom.xml +++ b/sdk/cosmos/azure-cosmos-encryption/pom.xml @@ -221,6 +221,9 @@ Licensed under the MIT License. unit true + 1 + 256 + paranoid %regex[.*] @@ -274,6 +277,9 @@ Licensed under the MIT License. true + 1 + 256 + paranoid @@ -298,6 +304,9 @@ Licensed under the MIT License. true + 1 + 256 + paranoid diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java index 99d5a2e8683f..f9ef8084dd6a 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java @@ -166,7 +166,7 @@ private void onAfterClassCore(ITestClass testClass) { } String msg = "NETTY LEAKS detected in test class: " - + this.getClass().getCanonicalName() + + testClassName + sb; logger.error(msg); @@ -206,11 +206,12 @@ public static void ingestIntoNetty() { return; } - // Must run before any Netty ByteBuf is allocated - ResourceLeakDetector.setLevel(ResourceLeakDetector.Level.PARANOID); // sample every allocation System.setProperty("io.netty.leakDetection.samplingInterval", "1"); System.setProperty("io.netty.leakDetection.targetRecords", "256"); + // Must run before any Netty ByteBuf is allocated + ResourceLeakDetector.setLevel(ResourceLeakDetector.Level.PARANOID); + // install custom reporter ResourceLeakDetectorFactory.setResourceLeakDetectorFactory(new CosmosNettyLeakDetectorFactory()); diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml b/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml index 3de211365b3e..f3663f541eef 100644 --- a/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml +++ b/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml @@ -278,6 +278,9 @@ Licensed under the MIT License. unit true + 1 + 256 + paranoid %regex[.*] @@ -542,6 +545,9 @@ Licensed under the MIT License. true + 1 + 256 + paranoid @@ -566,6 +572,9 @@ Licensed under the MIT License. true + 1 + 256 + paranoid @@ -590,6 +599,9 @@ Licensed under the MIT License. true + 1 + 256 + paranoid @@ -614,6 +626,9 @@ Licensed under the MIT License. true + 1 + 256 + paranoid diff --git a/sdk/cosmos/azure-cosmos-spark-account-data-resolver-sample/pom.xml b/sdk/cosmos/azure-cosmos-spark-account-data-resolver-sample/pom.xml index 88d2e504a2c6..a9c332d7af1c 100644 --- a/sdk/cosmos/azure-cosmos-spark-account-data-resolver-sample/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark-account-data-resolver-sample/pom.xml @@ -698,6 +698,9 @@ true + 1 + 256 + paranoid **/*.* diff --git a/sdk/cosmos/azure-cosmos-spark_3-3_2-12/pom.xml b/sdk/cosmos/azure-cosmos-spark_3-3_2-12/pom.xml index 53a15d614595..7b797b4cd36f 100644 --- a/sdk/cosmos/azure-cosmos-spark_3-3_2-12/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark_3-3_2-12/pom.xml @@ -120,6 +120,9 @@ true + 1 + 256 + paranoid **/*.* diff --git a/sdk/cosmos/azure-cosmos-spark_3-4_2-12/pom.xml b/sdk/cosmos/azure-cosmos-spark_3-4_2-12/pom.xml index b0639b663d22..afaa80a9e50f 100644 --- a/sdk/cosmos/azure-cosmos-spark_3-4_2-12/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark_3-4_2-12/pom.xml @@ -120,6 +120,9 @@ true + 1 + 256 + paranoid **/*.* diff --git a/sdk/cosmos/azure-cosmos-spark_3-5_2-12/pom.xml b/sdk/cosmos/azure-cosmos-spark_3-5_2-12/pom.xml index ef0c91ea9914..a8ffcf3b2d5d 100644 --- a/sdk/cosmos/azure-cosmos-spark_3-5_2-12/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark_3-5_2-12/pom.xml @@ -120,6 +120,9 @@ true + 1 + 256 + paranoid **/*.* diff --git a/sdk/cosmos/azure-cosmos-spark_3/pom.xml b/sdk/cosmos/azure-cosmos-spark_3/pom.xml index a58d1d6dd840..58e12cd940f0 100644 --- a/sdk/cosmos/azure-cosmos-spark_3/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark_3/pom.xml @@ -749,6 +749,9 @@ true + 1 + 256 + paranoid **/*.* diff --git a/sdk/cosmos/azure-cosmos-test/pom.xml b/sdk/cosmos/azure-cosmos-test/pom.xml index f87510f6525e..fe5dee23f566 100644 --- a/sdk/cosmos/azure-cosmos-test/pom.xml +++ b/sdk/cosmos/azure-cosmos-test/pom.xml @@ -154,6 +154,9 @@ Licensed under the MIT License. unit true + 1 + 256 + paranoid %regex[.*] diff --git a/sdk/cosmos/azure-cosmos-tests/pom.xml b/sdk/cosmos/azure-cosmos-tests/pom.xml index 156467069d05..e5898d7c8f8b 100644 --- a/sdk/cosmos/azure-cosmos-tests/pom.xml +++ b/sdk/cosmos/azure-cosmos-tests/pom.xml @@ -235,6 +235,9 @@ Licensed under the MIT License. unit true + 1 + 256 + paranoid %regex[.*] @@ -323,6 +326,9 @@ Licensed under the MIT License. true + 1 + 256 + paranoid @@ -347,6 +353,9 @@ Licensed under the MIT License. true + 1 + 256 + paranoid @@ -371,6 +380,9 @@ Licensed under the MIT License. true + 1 + 256 + paranoid @@ -395,6 +407,9 @@ Licensed under the MIT License. true + 1 + 256 + paranoid @@ -419,6 +434,9 @@ Licensed under the MIT License. true + 1 + 256 + paranoid @@ -443,6 +461,9 @@ Licensed under the MIT License. true + 1 + 256 + paranoid @@ -467,6 +488,9 @@ Licensed under the MIT License. true + 1 + 256 + paranoid @@ -491,6 +515,9 @@ Licensed under the MIT License. true + 1 + 256 + paranoid @@ -515,6 +542,9 @@ Licensed under the MIT License. true + 1 + 256 + paranoid @@ -539,6 +569,9 @@ Licensed under the MIT License. true + 1 + 256 + paranoid @@ -563,6 +596,9 @@ Licensed under the MIT License. true + 1 + 256 + paranoid @@ -587,6 +623,9 @@ Licensed under the MIT License. true + 1 + 256 + paranoid @@ -611,6 +650,9 @@ Licensed under the MIT License. true + 1 + 256 + paranoid @@ -635,6 +677,9 @@ Licensed under the MIT License. true + 1 + 256 + paranoid @@ -660,6 +705,9 @@ Licensed under the MIT License. true + 1 + 256 + paranoid @@ -692,6 +740,9 @@ Licensed under the MIT License. true + 1 + 256 + paranoid @@ -716,6 +767,9 @@ Licensed under the MIT License. true + 1 + 256 + paranoid @@ -740,6 +794,9 @@ Licensed under the MIT License. true + 1 + 256 + paranoid @@ -764,6 +821,9 @@ Licensed under the MIT License. true + 1 + 256 + paranoid diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java index eccf0760461f..30313c88ac03 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java @@ -206,11 +206,12 @@ public static void ingestIntoNetty() { return; } - // Must run before any Netty ByteBuf is allocated - ResourceLeakDetector.setLevel(ResourceLeakDetector.Level.PARANOID); // sample every allocation System.setProperty("io.netty.leakDetection.samplingInterval", "1"); System.setProperty("io.netty.leakDetection.targetRecords", "256"); + // Must run before any Netty ByteBuf is allocated + ResourceLeakDetector.setLevel(ResourceLeakDetector.Level.PARANOID); + // install custom reporter ResourceLeakDetectorFactory.setResourceLeakDetectorFactory(new CosmosNettyLeakDetectorFactory()); diff --git a/sdk/cosmos/azure-cosmos/pom.xml b/sdk/cosmos/azure-cosmos/pom.xml index f28a853bd9f9..c11f2a13b1e0 100644 --- a/sdk/cosmos/azure-cosmos/pom.xml +++ b/sdk/cosmos/azure-cosmos/pom.xml @@ -148,6 +148,9 @@ Licensed under the MIT License. unit true + 1 + 256 + paranoid %regex[.*] diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java index 25f02086acc3..37b4f56dbd3c 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java @@ -1380,6 +1380,9 @@ private Flux> createQueryInternal( private void addToActiveClients() { if (Configs.isClientLeakDetectionEnabled()) { + logger.warn( + "Cosmos Client leak detection is enabled - " + + "this will impact performance negatively - disable it in production scenarios."); synchronized (staticLock) { activeClients.put(this.clientId, StackTraceUtil.currentCallStack()); } diff --git a/sdk/cosmos/fabric-cosmos-spark-auth_3/pom.xml b/sdk/cosmos/fabric-cosmos-spark-auth_3/pom.xml index eb2d31982ffd..75035c197bc1 100644 --- a/sdk/cosmos/fabric-cosmos-spark-auth_3/pom.xml +++ b/sdk/cosmos/fabric-cosmos-spark-auth_3/pom.xml @@ -508,6 +508,9 @@ true + 1 + 256 + paranoid **/*.* From 2b4dc189411526439ce1fd8faf0290afac51103a Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Wed, 12 Nov 2025 11:19:45 +0000 Subject: [PATCH 39/65] Update CosmosNettyLeakDetectorFactory.java --- .../java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java index 30313c88ac03..cad4a4f0046a 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java @@ -166,7 +166,7 @@ private void onAfterClassCore(ITestClass testClass) { } String msg = "NETTY LEAKS detected in test class: " - + this.getClass().getCanonicalName() + + testClassName + sb; logger.error(msg); From c7e7360cca0a7614c73714368c35a30df5c82a37 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Wed, 12 Nov 2025 19:59:44 +0000 Subject: [PATCH 40/65] Test changes --- .../CosmosNettyLeakDetectorFactory.java | 23 +- .../azure/cosmos/CosmosDiagnosticsTest.java | 109 ++++---- .../CosmosNettyLeakDetectorFactory.java | 23 +- .../EndToEndTimeOutValidationTests.java | 16 +- .../DocumentQuerySpyWireContentTest.java | 16 +- .../cosmos/implementation/SessionTest.java | 31 ++- .../cosmos/implementation/TestSuiteBase.java | 240 +++++++++--------- .../cosmos/rx/ClientRetryPolicyE2ETests.java | 59 ++--- 8 files changed, 279 insertions(+), 238 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java index f9ef8084dd6a..2acab90cd4b2 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java @@ -27,6 +27,8 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; +import static org.assertj.core.api.Fail.fail; + public final class CosmosNettyLeakDetectorFactory extends ResourceLeakDetectorFactory implements IExecutionListener, IInvokedMethodListener, IClassListener { @@ -131,6 +133,7 @@ private void onAfterClassCore(ITestClass testClass) { int remainingInstanceCount = instanceCountSnapshot.decrementAndGet(); if (remainingInstanceCount == 0) { + String failMessage = ""; logger.info("LEAK DETECTION EVALUATION for test class {}", testClassName); Map leakedClientSnapshotNow = RxDocumentClientImpl.getActiveClientsSnapshot(); StringBuilder sb = new StringBuilder(); @@ -149,13 +152,10 @@ private void onAfterClassCore(ITestClass testClass) { } if (sb.length() > 0) { - String msg = "COSMOS CLIENT LEAKS detected in test class: " + failMessage = "COSMOS CLIENT LEAKS detected in test class: " + testClassName + "\n\n" + sb; - - logger.error(msg); - // fail(msg); } List nettyLeaks = CosmosNettyLeakDetectorFactory.resetIdentifiedLeaks(); @@ -165,13 +165,22 @@ private void onAfterClassCore(ITestClass testClass) { sb.append(leak).append("\n"); } - String msg = "NETTY LEAKS detected in test class: " + if (failMessage.length() > 0) { + failMessage += "\n\n"; + } + + failMessage += "NETTY LEAKS detected in test class: " + testClassName + + "\n\n" + sb; - logger.error(msg); - // fail(msg); } + + if (failMessage.length() > 0) { + logger.error(failMessage); + fail(failMessage); + } + this.logMemoryUsage("AFTER CLASS", testClassName); } } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosDiagnosticsTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosDiagnosticsTest.java index 5eb5b59867db..85a714e7af80 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosDiagnosticsTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosDiagnosticsTest.java @@ -328,41 +328,42 @@ public void queryChangeFeedIncrementalGatewayMode() throws Exception { @Test(groups = {"fast"}, timeOut = TIMEOUT) public void gatewayDiagnostics() throws Exception { - CosmosClient testClient = new CosmosClientBuilder() + try (CosmosClient testClient = new CosmosClientBuilder() .endpoint(TestConfigurations.HOST) .key(TestConfigurations.MASTER_KEY) .contentResponseOnWriteEnabled(true) .userAgentSuffix(USER_AGENT_SUFFIX_GATEWAY_CLIENT) .gatewayMode() - .buildClient(); + .buildClient()) { - CosmosContainer testContainer = - testClient - .getDatabase(cosmosAsyncContainer.getDatabase().getId()) - .getContainer(cosmosAsyncContainer.getId()); - // Adding a delay to allow async VM instance metadata initialization to complete - Thread.sleep(2000); - InternalObjectNode internalObjectNode = getInternalObjectNode(); - CosmosItemResponse createResponse = testContainer.createItem(internalObjectNode); - String diagnostics = createResponse.getDiagnostics().toString(); - logger.info("DIAGNOSTICS: {}", diagnostics); - assertThat(diagnostics).contains("\"connectionMode\":\"GATEWAY\""); - assertThat(diagnostics).contains("gatewayStatisticsList"); - assertThat(diagnostics).contains("\"operationType\":\"Create\""); - assertThat(diagnostics).contains("\"metaDataName\":\"CONTAINER_LOOK_UP\""); - assertThat(diagnostics).contains("\"serializationType\":\"PARTITION_KEY_FETCH_SERIALIZATION\""); - assertThat(diagnostics).contains("\"userAgent\":\"" + this.gatewayClientUserAgent + "\""); - assertThat(diagnostics).containsAnyOf( - "\"machineId\":\"" + tempMachineId + "\"", // logged machineId should be static uuid or - "\"machineId\":\"" + ClientTelemetry.getMachineId(null) + "\"" // the vmId from Azure - ); - assertThat(diagnostics).containsPattern("(?s).*?\"activityId\":\"[^\\s\"]+\".*"); - assertThat(createResponse.getDiagnostics().getDuration()).isNotNull(); - assertThat(createResponse.getDiagnostics().getContactedRegionNames()).isNotNull(); - assertThat(createResponse.getDiagnostics().getRegionsContacted()).isNotEmpty(); - validateTransportRequestTimelineGateway(diagnostics); - validateRegionContacted(createResponse.getDiagnostics(), gatewayClient.asyncClient()); - isValidJSON(diagnostics); + CosmosContainer testContainer = + testClient + .getDatabase(cosmosAsyncContainer.getDatabase().getId()) + .getContainer(cosmosAsyncContainer.getId()); + // Adding a delay to allow async VM instance metadata initialization to complete + Thread.sleep(2000); + InternalObjectNode internalObjectNode = getInternalObjectNode(); + CosmosItemResponse createResponse = testContainer.createItem(internalObjectNode); + String diagnostics = createResponse.getDiagnostics().toString(); + logger.info("DIAGNOSTICS: {}", diagnostics); + assertThat(diagnostics).contains("\"connectionMode\":\"GATEWAY\""); + assertThat(diagnostics).contains("gatewayStatisticsList"); + assertThat(diagnostics).contains("\"operationType\":\"Create\""); + assertThat(diagnostics).contains("\"metaDataName\":\"CONTAINER_LOOK_UP\""); + assertThat(diagnostics).contains("\"serializationType\":\"PARTITION_KEY_FETCH_SERIALIZATION\""); + assertThat(diagnostics).contains("\"userAgent\":\"" + this.gatewayClientUserAgent + "\""); + assertThat(diagnostics).containsAnyOf( + "\"machineId\":\"" + tempMachineId + "\"", // logged machineId should be static uuid or + "\"machineId\":\"" + ClientTelemetry.getMachineId(null) + "\"" // the vmId from Azure + ); + assertThat(diagnostics).containsPattern("(?s).*?\"activityId\":\"[^\\s\"]+\".*"); + assertThat(createResponse.getDiagnostics().getDuration()).isNotNull(); + assertThat(createResponse.getDiagnostics().getContactedRegionNames()).isNotNull(); + assertThat(createResponse.getDiagnostics().getRegionsContacted()).isNotEmpty(); + validateTransportRequestTimelineGateway(diagnostics); + validateRegionContacted(createResponse.getDiagnostics(), gatewayClient.asyncClient()); + isValidJSON(diagnostics); + } } @Test(groups = {"fast"}, timeOut = TIMEOUT) @@ -474,31 +475,32 @@ public void systemDiagnosticsForSystemStateInformation() { @Test(groups = {"fast"}, timeOut = TIMEOUT) public void directDiagnostics() throws Exception { - CosmosClient testClient = new CosmosClientBuilder() + try (CosmosClient testClient = new CosmosClientBuilder() .endpoint(TestConfigurations.HOST) .key(TestConfigurations.MASTER_KEY) .contentResponseOnWriteEnabled(true) .userAgentSuffix(USER_AGENT_SUFFIX_DIRECT_CLIENT) .directMode() - .buildClient(); + .buildClient()) { - CosmosContainer testContainer = - testClient - .getDatabase(cosmosAsyncContainer.getDatabase().getId()) - .getContainer(cosmosAsyncContainer.getId()); + CosmosContainer testContainer = + testClient + .getDatabase(cosmosAsyncContainer.getDatabase().getId()) + .getContainer(cosmosAsyncContainer.getId()); - InternalObjectNode internalObjectNode = getInternalObjectNode(); - CosmosItemResponse createResponse = testContainer.createItem(internalObjectNode); - validateDirectModeDiagnosticsOnSuccess(createResponse.getDiagnostics(), directClient, this.directClientUserAgent); - validateChannelAcquisitionContext(createResponse.getDiagnostics(), false); + InternalObjectNode internalObjectNode = getInternalObjectNode(); + CosmosItemResponse createResponse = testContainer.createItem(internalObjectNode); + validateDirectModeDiagnosticsOnSuccess(createResponse.getDiagnostics(), directClient, this.directClientUserAgent); + validateChannelAcquisitionContext(createResponse.getDiagnostics(), false); - // validate that on failed operation request timeline is populated - try { - testContainer.createItem(internalObjectNode); - fail("expected 409"); - } catch (CosmosException e) { - validateDirectModeDiagnosticsOnException(e, this.directClientUserAgent); - validateChannelAcquisitionContext(e.getDiagnostics(), false); + // validate that on failed operation request timeline is populated + try { + testContainer.createItem(internalObjectNode); + fail("expected 409"); + } catch (CosmosException e) { + validateDirectModeDiagnosticsOnException(e, this.directClientUserAgent); + validateChannelAcquisitionContext(e.getDiagnostics(), false); + } } } @@ -1808,12 +1810,15 @@ public void expireRecordWhenRecordAlreadyCompleteExceptionally() throws URISynta RxDocumentServiceRequest.create(mockDiagnosticsClientContext(), OperationType.Read, ResourceType.Document), new Uri(new URI("http://localhost/replica-path").toString()) ); - RntbdRequestTimer requestTimer = new RntbdRequestTimer(5000, 5000); - RntbdRequestRecord record = new AsyncRntbdRequestRecord(requestArgs, requestTimer); - record.completeExceptionally(exception); - // validate record.toString() will work correctly - String recordString = record.toString(); - assertThat(recordString.contains("NotFoundException")).isTrue(); + try (RntbdRequestTimer requestTimer = + new RntbdRequestTimer(5000, 5000)) { + + RntbdRequestRecord record = new AsyncRntbdRequestRecord(requestArgs, requestTimer); + record.completeExceptionally(exception); + // validate record.toString() will work correctly + String recordString = record.toString(); + assertThat(recordString.contains("NotFoundException")).isTrue(); + } } finally { safeClose(client); } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java index cad4a4f0046a..c37c6b8a0e6d 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java @@ -27,6 +27,8 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; +import static org.assertj.core.api.Fail.fail; + public final class CosmosNettyLeakDetectorFactory extends ResourceLeakDetectorFactory implements IExecutionListener, IInvokedMethodListener, IClassListener { @@ -131,6 +133,7 @@ private void onAfterClassCore(ITestClass testClass) { int remainingInstanceCount = instanceCountSnapshot.decrementAndGet(); if (remainingInstanceCount == 0) { + String failMessage = ""; logger.info("LEAK DETECTION EVALUATION for test class {}", testClassName); Map leakedClientSnapshotNow = RxDocumentClientImpl.getActiveClientsSnapshot(); StringBuilder sb = new StringBuilder(); @@ -149,13 +152,10 @@ private void onAfterClassCore(ITestClass testClass) { } if (sb.length() > 0) { - String msg = "COSMOS CLIENT LEAKS detected in test class: " + failMessage = "COSMOS CLIENT LEAKS detected in test class: " + testClassName + "\n\n" + sb; - - logger.error(msg); - // fail(msg); } List nettyLeaks = CosmosNettyLeakDetectorFactory.resetIdentifiedLeaks(); @@ -165,13 +165,22 @@ private void onAfterClassCore(ITestClass testClass) { sb.append(leak).append("\n"); } - String msg = "NETTY LEAKS detected in test class: " + if (failMessage.length() > 0) { + failMessage += "\n\n"; + } + + failMessage += "NETTY LEAKS detected in test class: " + testClassName + + "\n\n" + sb; - logger.error(msg); - // fail(msg); } + + if (failMessage.length() > 0) { + logger.error(failMessage); + fail(failMessage); + } + this.logMemoryUsage("AFTER CLASS", testClassName); } } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/EndToEndTimeOutValidationTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/EndToEndTimeOutValidationTests.java index e15d6a25ca0c..f1cc3e82a30c 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/EndToEndTimeOutValidationTests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/EndToEndTimeOutValidationTests.java @@ -25,6 +25,7 @@ import com.azure.cosmos.test.faultinjection.IFaultInjectionResult; import com.azure.cosmos.test.implementation.faultinjection.FaultInjectorProvider; import com.azure.cosmos.util.CosmosPagedFlux; +import net.bytebuddy.implementation.bytecode.Throw; import org.testng.SkipException; import org.testng.annotations.BeforeClass; import org.testng.annotations.Factory; @@ -60,12 +61,19 @@ public CosmosAsyncClient initializeClient(CosmosEndToEndOperationLatencyPolicyCo .getClientBuilder() .endToEndOperationLatencyPolicyConfig(e2eDefaultConfig) .buildAsyncClient(); - createdContainer = getSharedMultiPartitionCosmosContainer(client); - truncateCollection(createdContainer); - createdDocuments.addAll(this.insertDocuments(DEFAULT_NUM_DOCUMENTS, null, createdContainer)); + try { + createdContainer = getSharedMultiPartitionCosmosContainer(client); + truncateCollection(createdContainer); + + createdDocuments.addAll(this.insertDocuments(DEFAULT_NUM_DOCUMENTS, null, createdContainer)); - return client; + return client; + } catch (Throwable t) { + safeClose(client); + + throw t; + } } @Test(groups = {"fast"}, timeOut = 10000L, retryAnalyzer = FlakyTestRetryAnalyzer.class) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/DocumentQuerySpyWireContentTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/DocumentQuerySpyWireContentTest.java index 5540662212d8..22b3be737817 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/DocumentQuerySpyWireContentTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/DocumentQuerySpyWireContentTest.java @@ -182,13 +182,17 @@ public void before_DocumentQuerySpyWireContentTest() throws Exception { client ); - // do the query once to ensure the collection is cached. - client.queryDocuments(getMultiPartitionCollectionLink(), "select * from root", state, Document.class) - .then().block(); + try { + // do the query once to ensure the collection is cached. + client.queryDocuments(getMultiPartitionCollectionLink(), "select * from root", state, Document.class) + .then().block(); - // do the query once to ensure the collection is cached. - client.queryDocuments(getSinglePartitionCollectionLink(), "select * from root", state, Document.class) - .then().block(); + // do the query once to ensure the collection is cached. + client.queryDocuments(getSinglePartitionCollectionLink(), "select * from root", state, Document.class) + .then().block(); + } finally { + safeClose(state); + } } @AfterClass(groups = { "fast" }, timeOut = SHUTDOWN_TIMEOUT, alwaysRun = true) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/SessionTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/SessionTest.java index ee03e5e34414..e672381f247a 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/SessionTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/SessionTest.java @@ -78,21 +78,26 @@ public void before_SessionTest() { RequestOptions requestOptions = new RequestOptions(); requestOptions.setOfferThroughput(20000); //Making sure we have 4 physical partitions - createdCollection = createCollection(createGatewayHouseKeepingDocumentClient().build(), createdDatabase.getId(), + AsyncDocumentClient asynClient = createGatewayHouseKeepingDocumentClient().build(); + try { + createdCollection = createCollection(asynClient, createdDatabase.getId(), collection, requestOptions); - houseKeepingClient = clientBuilder().build(); - connectionMode = houseKeepingClient.getConnectionPolicy().getConnectionMode(); - - if (connectionMode == ConnectionMode.DIRECT) { - spyClient = SpyClientUnderTestFactory.createDirectHttpsClientUnderTest(clientBuilder()); - } else { - // Gateway builder has multipleWriteRegionsEnabled false by default, enabling it for multi master test - ConnectionPolicy connectionPolicy = clientBuilder().connectionPolicy; - connectionPolicy.setMultipleWriteRegionsEnabled(true); - spyClient = SpyClientUnderTestFactory.createClientUnderTest(clientBuilder().withConnectionPolicy(connectionPolicy)); + houseKeepingClient = clientBuilder().build(); + connectionMode = houseKeepingClient.getConnectionPolicy().getConnectionMode(); + + if (connectionMode == ConnectionMode.DIRECT) { + spyClient = SpyClientUnderTestFactory.createDirectHttpsClientUnderTest(clientBuilder()); + } else { + // Gateway builder has multipleWriteRegionsEnabled false by default, enabling it for multi master test + ConnectionPolicy connectionPolicy = clientBuilder().connectionPolicy; + connectionPolicy.setMultipleWriteRegionsEnabled(true); + spyClient = SpyClientUnderTestFactory.createClientUnderTest(clientBuilder().withConnectionPolicy(connectionPolicy)); + } + options = new RequestOptions(); + options.setPartitionKey(PartitionKey.NONE); + } finally { + asynClient.close(); } - options = new RequestOptions(); - options.setPartitionKey(PartitionKey.NONE); } @AfterClass(groups = { "fast", "multi-master" }, timeOut = SHUTDOWN_TIMEOUT, alwaysRun = true) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestSuiteBase.java index 2a977e4159fd..ce0b29a47de5 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/TestSuiteBase.java @@ -179,128 +179,128 @@ protected static void truncateCollection(DocumentCollection collection) { try { List paths = collection.getPartitionKey().getPaths(); - CosmosAsyncClient cosmosClient = new CosmosClientBuilder() + try (CosmosAsyncClient cosmosClient = new CosmosClientBuilder() .key(TestConfigurations.MASTER_KEY) .endpoint(TestConfigurations.HOST) - .buildAsyncClient(); - CosmosQueryRequestOptions options = new CosmosQueryRequestOptions(); - options.setMaxDegreeOfParallelism(-1); - QueryFeedOperationState state = new QueryFeedOperationState( - cosmosClient, - "truncateCollection", - collection.getSelfLink(), - collection.getId(), - ResourceType.Document, - OperationType.Query, - null, - options, - new CosmosPagedFluxOptions() - ); - - ModelBridgeInternal.setQueryRequestOptionsMaxItemCount(options, 100); - - logger.info("Truncating DocumentCollection {} documents ...", collection.getId()); - - houseKeepingClient.queryDocuments(collection.getSelfLink(), "SELECT * FROM root", state, Document.class) - .publishOn(Schedulers.parallel()) - .flatMap(page -> Flux.fromIterable(page.getResults())) - .flatMap(doc -> { - RequestOptions requestOptions = new RequestOptions(); - - if (paths != null && !paths.isEmpty()) { - List pkPath = PathParser.getPathParts(paths.get(0)); - Object propertyValue = doc.getObjectByPath(pkPath); - if (propertyValue == null) { - propertyValue = Undefined.value(); - } - - requestOptions.setPartitionKey(new PartitionKey(propertyValue)); - } - - return houseKeepingClient.deleteDocument(doc.getSelfLink(), requestOptions); - }).then().block(); - - logger.info("Truncating DocumentCollection {} triggers ...", collection.getId()); - - state = new QueryFeedOperationState( - cosmosClient, - "truncateTriggers", - collection.getSelfLink(), - collection.getId(), - ResourceType.Document, - OperationType.Query, - null, - options, - new CosmosPagedFluxOptions() - ); - houseKeepingClient.queryTriggers(collection.getSelfLink(), "SELECT * FROM root", state) - .publishOn(Schedulers.parallel()) - .flatMap(page -> Flux.fromIterable(page.getResults())) - .flatMap(trigger -> { - RequestOptions requestOptions = new RequestOptions(); - -// if (paths != null && !paths.isEmpty()) { -// Object propertyValue = trigger.getObjectByPath(PathParser.getPathParts(paths.get(0))); -// requestOptions.partitionKey(new PartitionKey(propertyValue)); -// } - - return houseKeepingClient.deleteTrigger(trigger.getSelfLink(), requestOptions); - }).then().block(); - - logger.info("Truncating DocumentCollection {} storedProcedures ...", collection.getId()); - - state = new QueryFeedOperationState( - cosmosClient, - "truncateStoredProcs", - collection.getSelfLink(), - collection.getId(), - ResourceType.Document, - OperationType.Query, - null, - options, - new CosmosPagedFluxOptions() - ); - houseKeepingClient.queryStoredProcedures(collection.getSelfLink(), "SELECT * FROM root", state) - .publishOn(Schedulers.parallel()) - .flatMap(page -> Flux.fromIterable(page.getResults())) - .flatMap(storedProcedure -> { - RequestOptions requestOptions = new RequestOptions(); - -// if (paths != null && !paths.isEmpty()) { -// Object propertyValue = storedProcedure.getObjectByPath(PathParser.getPathParts(paths.get(0))); -// requestOptions.partitionKey(new PartitionKey(propertyValue)); -// } - - return houseKeepingClient.deleteStoredProcedure(storedProcedure.getSelfLink(), requestOptions); - }).then().block(); - - logger.info("Truncating DocumentCollection {} udfs ...", collection.getId()); - - state = new QueryFeedOperationState( - cosmosClient, - "truncateUserDefinedFunctions", - collection.getSelfLink(), - collection.getId(), - ResourceType.Document, - OperationType.Query, - null, - options, - new CosmosPagedFluxOptions() - ); - houseKeepingClient.queryUserDefinedFunctions(collection.getSelfLink(), "SELECT * FROM root", state) - .publishOn(Schedulers.parallel()) - .flatMap(page -> Flux.fromIterable(page.getResults())) - .flatMap(udf -> { - RequestOptions requestOptions = new RequestOptions(); - -// if (paths != null && !paths.isEmpty()) { -// Object propertyValue = udf.getObjectByPath(PathParser.getPathParts(paths.get(0))); -// requestOptions.partitionKey(new PartitionKey(propertyValue)); -// } - - return houseKeepingClient.deleteUserDefinedFunction(udf.getSelfLink(), requestOptions); - }).then().block(); - + .buildAsyncClient()) { + CosmosQueryRequestOptions options = new CosmosQueryRequestOptions(); + options.setMaxDegreeOfParallelism(-1); + QueryFeedOperationState state = new QueryFeedOperationState( + cosmosClient, + "truncateCollection", + collection.getSelfLink(), + collection.getId(), + ResourceType.Document, + OperationType.Query, + null, + options, + new CosmosPagedFluxOptions() + ); + + ModelBridgeInternal.setQueryRequestOptionsMaxItemCount(options, 100); + + logger.info("Truncating DocumentCollection {} documents ...", collection.getId()); + + houseKeepingClient.queryDocuments(collection.getSelfLink(), "SELECT * FROM root", state, Document.class) + .publishOn(Schedulers.parallel()) + .flatMap(page -> Flux.fromIterable(page.getResults())) + .flatMap(doc -> { + RequestOptions requestOptions = new RequestOptions(); + + if (paths != null && !paths.isEmpty()) { + List pkPath = PathParser.getPathParts(paths.get(0)); + Object propertyValue = doc.getObjectByPath(pkPath); + if (propertyValue == null) { + propertyValue = Undefined.value(); + } + + requestOptions.setPartitionKey(new PartitionKey(propertyValue)); + } + + return houseKeepingClient.deleteDocument(doc.getSelfLink(), requestOptions); + }).then().block(); + + logger.info("Truncating DocumentCollection {} triggers ...", collection.getId()); + + state = new QueryFeedOperationState( + cosmosClient, + "truncateTriggers", + collection.getSelfLink(), + collection.getId(), + ResourceType.Document, + OperationType.Query, + null, + options, + new CosmosPagedFluxOptions() + ); + houseKeepingClient.queryTriggers(collection.getSelfLink(), "SELECT * FROM root", state) + .publishOn(Schedulers.parallel()) + .flatMap(page -> Flux.fromIterable(page.getResults())) + .flatMap(trigger -> { + RequestOptions requestOptions = new RequestOptions(); + + // if (paths != null && !paths.isEmpty()) { + // Object propertyValue = trigger.getObjectByPath(PathParser.getPathParts(paths.get(0))); + // requestOptions.partitionKey(new PartitionKey(propertyValue)); + // } + + return houseKeepingClient.deleteTrigger(trigger.getSelfLink(), requestOptions); + }).then().block(); + + logger.info("Truncating DocumentCollection {} storedProcedures ...", collection.getId()); + + state = new QueryFeedOperationState( + cosmosClient, + "truncateStoredProcs", + collection.getSelfLink(), + collection.getId(), + ResourceType.Document, + OperationType.Query, + null, + options, + new CosmosPagedFluxOptions() + ); + houseKeepingClient.queryStoredProcedures(collection.getSelfLink(), "SELECT * FROM root", state) + .publishOn(Schedulers.parallel()) + .flatMap(page -> Flux.fromIterable(page.getResults())) + .flatMap(storedProcedure -> { + RequestOptions requestOptions = new RequestOptions(); + + // if (paths != null && !paths.isEmpty()) { + // Object propertyValue = storedProcedure.getObjectByPath(PathParser.getPathParts(paths.get(0))); + // requestOptions.partitionKey(new PartitionKey(propertyValue)); + // } + + return houseKeepingClient.deleteStoredProcedure(storedProcedure.getSelfLink(), requestOptions); + }).then().block(); + + logger.info("Truncating DocumentCollection {} udfs ...", collection.getId()); + + state = new QueryFeedOperationState( + cosmosClient, + "truncateUserDefinedFunctions", + collection.getSelfLink(), + collection.getId(), + ResourceType.Document, + OperationType.Query, + null, + options, + new CosmosPagedFluxOptions() + ); + houseKeepingClient.queryUserDefinedFunctions(collection.getSelfLink(), "SELECT * FROM root", state) + .publishOn(Schedulers.parallel()) + .flatMap(page -> Flux.fromIterable(page.getResults())) + .flatMap(udf -> { + RequestOptions requestOptions = new RequestOptions(); + + // if (paths != null && !paths.isEmpty()) { + // Object propertyValue = udf.getObjectByPath(PathParser.getPathParts(paths.get(0))); + // requestOptions.partitionKey(new PartitionKey(propertyValue)); + // } + + return houseKeepingClient.deleteUserDefinedFunction(udf.getSelfLink(), requestOptions); + }).then().block(); + } } finally { houseKeepingClient.close(); } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ClientRetryPolicyE2ETests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ClientRetryPolicyE2ETests.java index 8d4e7fe2570b..66e22a8fdbef 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ClientRetryPolicyE2ETests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ClientRetryPolicyE2ETests.java @@ -57,7 +57,6 @@ import reactor.core.publisher.Mono; import java.time.Duration; -import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -68,7 +67,6 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import java.util.stream.Collectors; @@ -154,46 +152,49 @@ public ClientRetryPolicyE2ETests(CosmosClientBuilder clientBuilder) { @BeforeClass(groups = {"multi-master", "fast", "fi-multi-master", "multi-region"}, timeOut = TIMEOUT) public void beforeClass() { - CosmosAsyncClient dummy = getClientBuilder().buildAsyncClient(); - AsyncDocumentClient asyncDocumentClient = BridgeInternal.getContextClient(dummy); - GlobalEndpointManager globalEndpointManager = asyncDocumentClient.getGlobalEndpointManager(); + try(CosmosAsyncClient dummy = getClientBuilder().buildAsyncClient()) { + AsyncDocumentClient asyncDocumentClient = BridgeInternal.getContextClient(dummy); + GlobalEndpointManager globalEndpointManager = asyncDocumentClient.getGlobalEndpointManager(); - DatabaseAccount databaseAccount = globalEndpointManager.getLatestDatabaseAccount(); + DatabaseAccount databaseAccount = globalEndpointManager.getLatestDatabaseAccount(); - AccountLevelLocationContext accountLevelReadableLocationContext - = getAccountLevelLocationContext(databaseAccount, false); + AccountLevelLocationContext accountLevelReadableLocationContext + = getAccountLevelLocationContext(databaseAccount, false); - AccountLevelLocationContext accountLevelWriteableLocationContext - = getAccountLevelLocationContext(databaseAccount, true); + AccountLevelLocationContext accountLevelWriteableLocationContext + = getAccountLevelLocationContext(databaseAccount, true); - validate(accountLevelReadableLocationContext, false); - validate(accountLevelWriteableLocationContext, true); + validate(accountLevelReadableLocationContext, false); + validate(accountLevelWriteableLocationContext, true); - this.preferredRegions = accountLevelReadableLocationContext.serviceOrderedReadableRegions + this.preferredRegions = accountLevelReadableLocationContext.serviceOrderedReadableRegions .stream() .map(regionName -> regionName.toLowerCase(Locale.ROOT)) .collect(Collectors.toList()); - this.serviceOrderedReadableRegions = this.preferredRegions; + this.serviceOrderedReadableRegions = this.preferredRegions; - this.serviceOrderedWriteableRegions = accountLevelWriteableLocationContext.serviceOrderedWriteableRegions - .stream() - .map(regionName -> regionName.toLowerCase(Locale.ROOT)) - .collect(Collectors.toList()); + this.serviceOrderedWriteableRegions = accountLevelWriteableLocationContext.serviceOrderedWriteableRegions + .stream() + .map(regionName -> regionName.toLowerCase(Locale.ROOT)) + .collect(Collectors.toList()); - this.clientWithPreferredRegions = getClientBuilder() - .preferredRegions(this.preferredRegions) - .endpointDiscoveryEnabled(true) - .multipleWriteRegionsEnabled(true) - .buildAsyncClient(); + this.clientWithPreferredRegions = getClientBuilder() + .preferredRegions(this.preferredRegions) + .endpointDiscoveryEnabled(true) + .multipleWriteRegionsEnabled(true) + .buildAsyncClient(); - this.clientWithoutPreferredRegions = getClientBuilder() - .endpointDiscoveryEnabled(true) - .multipleWriteRegionsEnabled(true) - .buildAsyncClient(); + this.clientWithoutPreferredRegions = getClientBuilder() + .endpointDiscoveryEnabled(true) + .multipleWriteRegionsEnabled(true) + .buildAsyncClient(); + } - this.cosmosAsyncContainerFromClientWithPreferredRegions = getSharedMultiPartitionCosmosContainerWithIdAsPartitionKey(clientWithPreferredRegions); - this.cosmosAsyncContainerFromClientWithoutPreferredRegions = getSharedMultiPartitionCosmosContainerWithIdAsPartitionKey(clientWithoutPreferredRegions); + this.cosmosAsyncContainerFromClientWithPreferredRegions = + getSharedMultiPartitionCosmosContainerWithIdAsPartitionKey(clientWithPreferredRegions); + this.cosmosAsyncContainerFromClientWithoutPreferredRegions = + getSharedMultiPartitionCosmosContainerWithIdAsPartitionKey(clientWithoutPreferredRegions); } @AfterClass(groups = {"multi-master", "fast", "fi-multi-master", "multi-region"}, timeOut = SHUTDOWN_TIMEOUT, alwaysRun = true) From 3fd46ae9a29fdddf0c355252009b6ec26ce4c1ad Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Thu, 13 Nov 2025 00:12:54 +0000 Subject: [PATCH 41/65] Prod memory leak fixes --- .../implementation/RxGatewayStoreModel.java | 5 +- .../implementation/ThinClientStoreModel.java | 98 +++++++++++-------- .../WebExceptionRetryPolicy.java | 15 ++- .../http/ReactorNettyClient.java | 15 ++- 4 files changed, 82 insertions(+), 51 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java index 82d7aeba54c3..680b591d38c9 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java @@ -36,6 +36,7 @@ import io.netty.buffer.Unpooled; import io.netty.handler.codec.http.HttpMethod; import io.netty.handler.codec.http.HttpResponseStatus; +import io.netty.util.ReferenceCountUtil; import io.netty.util.ResourceLeakDetector; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -422,7 +423,7 @@ private Mono toDocumentServiceResponse(Mono 0) { // there could be a race with the catch in the .map operator below // so, use safeRelease - io.netty.util.ReferenceCountUtil.safeRelease(buf); + ReferenceCountUtil.safeRelease(buf); } }); @@ -469,7 +470,7 @@ private Mono toDocumentServiceResponse(Mono 0) { // Unwrap failed before StoreResponse took ownership -> release our retain // there could be a race with the doOnDiscard above - so, use safeRelease - io.netty.util.ReferenceCountUtil.safeRelease(content); + ReferenceCountUtil.safeRelease(content); } throw t; diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ThinClientStoreModel.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ThinClientStoreModel.java index 3922ca187db7..1bd343e6192b 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ThinClientStoreModel.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ThinClientStoreModel.java @@ -15,8 +15,10 @@ import com.azure.cosmos.implementation.routing.HexConvert; import com.azure.cosmos.implementation.routing.PartitionKeyInternal; import io.netty.buffer.ByteBuf; +import io.netty.buffer.ByteBufUtil; import io.netty.buffer.Unpooled; import io.netty.handler.codec.http.HttpMethod; +import io.netty.util.ReferenceCountUtil; import io.netty.util.ResourceLeakDetector; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -36,7 +38,7 @@ public class ThinClientStoreModel extends RxGatewayStoreModel { private static final boolean leakDetectionDebuggingEnabled = ResourceLeakDetector.getLevel().ordinal() >= ResourceLeakDetector.Level.ADVANCED.ordinal(); - private String globalDatabaseAccountName = null; + private volatile String globalDatabaseAccountName = null; private final Map defaultHeaders; public ThinClientStoreModel( @@ -104,46 +106,56 @@ public StoreResponse unwrapToStoreResponse( if (content.readableBytes() == 0) { - content.release(); + ReferenceCountUtil.safeRelease(content); return super.unwrapToStoreResponse(endpoint, request, statusCode, headers, Unpooled.EMPTY_BUFFER); } if (leakDetectionDebuggingEnabled) { content.touch(this); } - if (RntbdFramer.canDecodeHead(content)) { - final RntbdResponse response = RntbdResponse.decode(content); - - if (response != null) { - ByteBuf payloadBuf = response.getContent(); - - if (payloadBuf != Unpooled.EMPTY_BUFFER && leakDetectionDebuggingEnabled) { - content.touch(this); + try { + if (RntbdFramer.canDecodeHead(content)) { + + final RntbdResponse response = RntbdResponse.decode(content); + + if (response != null) { + ByteBuf payloadBuf = response.getContent(); + + if (payloadBuf != Unpooled.EMPTY_BUFFER && leakDetectionDebuggingEnabled) { + payloadBuf.touch(this); + } + + try { + StoreResponse storeResponse = super.unwrapToStoreResponse( + endpoint, + request, + response.getStatus().code(), + new HttpHeaders(response.getHeaders().asMap(request.getActivityId())), + payloadBuf + ); + + return storeResponse; + } finally { + if (payloadBuf != content) { + // payload is a slice/derived view; super() owns payload, we still own the container + // this includes scenarios where payloadBuf == EMPTY_BUFFER + ReferenceCountUtil.safeRelease(content); + } + } } - StoreResponse storeResponse = super.unwrapToStoreResponse( - endpoint, - request, - response.getStatus().code(), - new HttpHeaders(response.getHeaders().asMap(request.getActivityId())), - payloadBuf - ); - - if (payloadBuf == Unpooled.EMPTY_BUFFER) { - // means the original RNTBD payload did not have any payload - so, we can release it - content.release(); - } - - return storeResponse; + ReferenceCountUtil.safeRelease(content); + return super.unwrapToStoreResponse(endpoint, request, statusCode, headers, Unpooled.EMPTY_BUFFER); } - content.release(); - return super.unwrapToStoreResponse(endpoint, request, statusCode, headers, null); + ReferenceCountUtil.safeRelease(content); + throw new IllegalStateException("Invalid rntbd response"); + } catch (Throwable t) { + // Ensure container is not leaked on any unexpected path + ReferenceCountUtil.safeRelease(content); + throw t; } - - content.release(); - throw new IllegalStateException("Invalid rntbd response"); } @Override @@ -157,7 +169,7 @@ public HttpRequest wrapInHttpRequest(RxDocumentServiceRequest request, URI reque if (this.globalDatabaseAccountName == null) { this.globalDatabaseAccountName = this.globalEndpointManager.getLatestDatabaseAccount().getId(); } - // todo - neharao1 - validate b/w name() v/s toString() + request.setThinclientHeaders( request.getOperationType(), request.getResourceType(), @@ -193,18 +205,20 @@ public HttpRequest wrapInHttpRequest(RxDocumentServiceRequest request, URI reque // todo: eventually need to use pooled buffer ByteBuf byteBuf = Unpooled.buffer(); - - rntbdRequest.encode(byteBuf, true); - - byte[] contentAsByteArray = new byte[byteBuf.writerIndex()]; - byteBuf.getBytes(0, contentAsByteArray, 0, byteBuf.writerIndex()); - - return new HttpRequest( - HttpMethod.POST, - requestUri, - requestUri.getPort(), - headers, - Flux.just(contentAsByteArray)); + try { + rntbdRequest.encode(byteBuf, true); + + byte[] contentAsByteArray = ByteBufUtil.getBytes(byteBuf, 0, byteBuf.writerIndex(), false); + + return new HttpRequest( + HttpMethod.POST, + requestUri, + requestUri.getPort(), + headers, + Flux.just(contentAsByteArray)); + } finally { + ReferenceCountUtil.safeRelease(byteBuf); + } } @Override diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/WebExceptionRetryPolicy.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/WebExceptionRetryPolicy.java index ab1452cd2537..b1f737530f31 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/WebExceptionRetryPolicy.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/WebExceptionRetryPolicy.java @@ -42,9 +42,18 @@ public WebExceptionRetryPolicy(RetryContext retryContext) { @Override public Mono shouldRetry(Exception e) { - checkArgument(this.overriddenEndpoint != null || - this.regionalRoutingContext != null, - "Both overriddenEndpoint and regionalRoutingContext cannot null!"); + // if both are null it means the client wasn't initialized yet + if (this.overriddenEndpoint == null && this.regionalRoutingContext == null) { + logger + .warn( + "WebExceptionRetryPolicy() No retries because client is not initialized yet. - " + + "operationType = {}, count = {}, isAddressRefresh = {}", + this.request.getOperationType(), + this.retryCount, + this.request.isAddressRefresh()); + + return Mono.just(ShouldRetryResult.noRetry()); + } if (this.isOutOfRetries()) { logger diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java index 1556a6648d2d..488596e825d5 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java @@ -346,14 +346,21 @@ public HttpHeaders headers() { @Override public Mono body() { - return bodyIntern() + return ByteBufFlux + .fromInbound( + bodyIntern().doOnDiscard(ByteBuf.class, io.netty.util.ReferenceCountUtil::safeRelease) + ) .aggregate() .doOnSubscribe(this::updateSubscriptionState); } @Override public Mono bodyAsString() { - return bodyIntern().aggregate() + return ByteBufFlux + .fromInbound( + bodyIntern().doOnDiscard(ByteBuf.class, io.netty.util.ReferenceCountUtil::safeRelease) + ) + .aggregate() .asString() .doOnSubscribe(this::updateSubscriptionState); } @@ -394,8 +401,8 @@ private void releaseOnNotSubscribedResponse(ReactorNettyResponseState reactorNet logger.debug("Releasing body, not yet subscribed"); } this.bodyIntern() - .doOnNext(byteBuf -> {}) - .subscribe(byteBuf -> {}, ex -> {}); + .doOnNext(io.netty.util.ReferenceCountUtil::safeRelease) + .subscribe(v -> {}, ex -> {}, () -> {}); } } } From 5c0d0219cf5de7947320e688d5a2de41226a038b Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Thu, 13 Nov 2025 00:14:34 +0000 Subject: [PATCH 42/65] Test fixes --- .../CosmosNettyLeakDetectorFactory.java | 16 +-- .../azure/cosmos/CosmosDiagnosticsTest.java | 2 +- .../CosmosNettyLeakDetectorFactory.java | 16 +-- .../GatewayAddressCacheTest.java | 2 +- .../azure/cosmos/rx/ReadFeedOffersTest.java | 48 ++++---- .../rx/proxy/HttpProxyClientHandler.java | 41 +++++-- .../rx/proxy/HttpProxyClientHeader.java | 107 ++++++++++++------ .../rx/proxy/HttpProxyRemoteHandler.java | 27 ++++- 8 files changed, 161 insertions(+), 98 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java index 2acab90cd4b2..279b3b671323 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java @@ -89,13 +89,7 @@ public void onAfterClass(ITestClass testClass) { ITestNGMethod[] afterClassMethods = testClass.getAfterClassMethods(); boolean testClassHasAfterClassMethods = afterClassMethods != null && afterClassMethods.length > 0; if (!testClassHasAfterClassMethods) { - try { - this.onAfterClassCore(testClass); - } catch (Throwable t) { - // decide if you want to fail the suite or just log - System.err.println("Symmetric afterClass failed for " + testClass.getRealClass().getCanonicalName()); - t.printStackTrace(); - } + this.onAfterClassCore(testClass); } } @@ -110,13 +104,7 @@ public void afterInvocation(IInvokedMethod method, ITestResult result) { && method.getTestMethod().isAfterClassConfiguration()) { // <-- This point is guaranteed to be AFTER the class’s @AfterClass ran if any existed - try { - this.onAfterClassCore(testClass); - } catch (Throwable t) { - // decide if you want to fail the suite or just log - System.err.println("Symmetric afterClass failed for " + testClass.getRealClass().getCanonicalName()); - t.printStackTrace(); - } + this.onAfterClassCore(testClass); } } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosDiagnosticsTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosDiagnosticsTest.java index 85a714e7af80..ab5d26f0d4c4 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosDiagnosticsTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosDiagnosticsTest.java @@ -798,7 +798,7 @@ public void queryDiagnosticsOnOrderBy() { deleteCollection(testcontainer); } - @Test(groups = {"fast"}, dataProvider = "operationTypeProvider", timeOut = TIMEOUT) + @Test(groups = {"fast"}, dataProvider = "operationTypeProvider", timeOut = TIMEOUT, retryAnalyzer = FlakyTestRetryAnalyzer.class) public void directDiagnosticsOnCancelledOperation(OperationType operationType) { CosmosAsyncClient client = null; diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java index c37c6b8a0e6d..919c0f2c5391 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java @@ -89,13 +89,7 @@ public void onAfterClass(ITestClass testClass) { ITestNGMethod[] afterClassMethods = testClass.getAfterClassMethods(); boolean testClassHasAfterClassMethods = afterClassMethods != null && afterClassMethods.length > 0; if (!testClassHasAfterClassMethods) { - try { - this.onAfterClassCore(testClass); - } catch (Throwable t) { - // decide if you want to fail the suite or just log - System.err.println("Symmetric afterClass failed for " + testClass.getRealClass().getCanonicalName()); - t.printStackTrace(); - } + this.onAfterClassCore(testClass); } } @@ -110,13 +104,7 @@ public void afterInvocation(IInvokedMethod method, ITestResult result) { && method.getTestMethod().isAfterClassConfiguration()) { // <-- This point is guaranteed to be AFTER the class’s @AfterClass ran if any existed - try { - this.onAfterClassCore(testClass); - } catch (Throwable t) { - // decide if you want to fail the suite or just log - System.err.println("Symmetric afterClass failed for " + testClass.getRealClass().getCanonicalName()); - t.printStackTrace(); - } + this.onAfterClassCore(testClass); } } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/GatewayAddressCacheTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/GatewayAddressCacheTest.java index c92781137a97..8285ea915603 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/GatewayAddressCacheTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/GatewayAddressCacheTest.java @@ -168,7 +168,7 @@ public void getServerAddressesViaGateway(List partitionKeyRangeIds, } } - @Test(groups = { "direct" }, dataProvider = "protocolProvider", timeOut = TIMEOUT) + @Test(groups = { "direct" }, dataProvider = "protocolProvider", timeOut = TIMEOUT, retryAnalyzer = FlakyTestRetryAnalyzer.class) @SuppressWarnings({"unchecked", "rawtypes"}) public void getMasterAddressesViaGatewayAsync(Protocol protocol) throws Exception { Configs configs = ConfigsBuilder.instance().withProtocol(protocol).build(); diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedOffersTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedOffersTest.java index 6bc22bd03a2d..69e4bb9f4a1a 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedOffersTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ReadFeedOffersTest.java @@ -4,6 +4,7 @@ import com.azure.cosmos.CosmosAsyncClient; import com.azure.cosmos.CosmosClientBuilder; +import com.azure.cosmos.FlakyTestRetryAnalyzer; import com.azure.cosmos.implementation.CosmosPagedFluxOptions; import com.azure.cosmos.implementation.OperationType; import com.azure.cosmos.implementation.QueryFeedOperationState; @@ -53,41 +54,42 @@ public ReadFeedOffersTest(AsyncDocumentClient.Builder clientBuilder) { super(clientBuilder); } - @Test(groups = { "query" }, timeOut = FEED_TIMEOUT) + @Test(groups = { "query" }, timeOut = FEED_TIMEOUT, retryAnalyzer = FlakyTestRetryAnalyzer.class) public void readOffers() throws Exception { CosmosQueryRequestOptions options = new CosmosQueryRequestOptions(); ModelBridgeInternal.setQueryRequestOptionsMaxItemCount(options, 2); - CosmosAsyncClient cosmosClient = new CosmosClientBuilder() + try (CosmosAsyncClient cosmosClient = new CosmosClientBuilder() .key(TestConfigurations.MASTER_KEY) .endpoint(TestConfigurations.HOST) - .buildAsyncClient(); - QueryFeedOperationState dummyState = new QueryFeedOperationState( - cosmosClient, - "SomeSpanName", - "SomeDBName", - "SomeContainerName", - ResourceType.Document, - OperationType.Query, - null, - options, - new CosmosPagedFluxOptions() - ); - - Flux> feedObservable = client.readOffers(dummyState); - - int maxItemCount = ModelBridgeInternal.getMaxItemCountFromQueryRequestOptions(options); - int expectedPageSize = (allOffers.size() + maxItemCount - 1) / maxItemCount; - - FeedResponseListValidator validator = new FeedResponseListValidator.Builder() + .buildAsyncClient()) { + QueryFeedOperationState dummyState = new QueryFeedOperationState( + cosmosClient, + "SomeSpanName", + "SomeDBName", + "SomeContainerName", + ResourceType.Document, + OperationType.Query, + null, + options, + new CosmosPagedFluxOptions() + ); + + Flux> feedObservable = client.readOffers(dummyState); + + int maxItemCount = ModelBridgeInternal.getMaxItemCountFromQueryRequestOptions(options); + int expectedPageSize = (allOffers.size() + maxItemCount - 1) / maxItemCount; + + FeedResponseListValidator validator = new FeedResponseListValidator.Builder() .totalSize(allOffers.size()) .exactlyContainsInAnyOrder(allOffers.stream().map(d -> d.getResourceId()).collect(Collectors.toList())) .numberOfPages(expectedPageSize) .pageSatisfy(0, new FeedResponseValidator.Builder() - .requestChargeGreaterThanOrEqualTo(1.0).build()) + .requestChargeGreaterThanOrEqualTo(1.0).build()) .build(); - validateQuerySuccess(feedObservable, validator, FEED_TIMEOUT); + validateQuerySuccess(feedObservable, validator, FEED_TIMEOUT); + } } @BeforeClass(groups = { "query" }, timeOut = SETUP_TIMEOUT) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/proxy/HttpProxyClientHandler.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/proxy/HttpProxyClientHandler.java index 79bfbc76336e..6114d930c1ec 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/proxy/HttpProxyClientHandler.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/proxy/HttpProxyClientHandler.java @@ -10,14 +10,22 @@ import io.netty.channel.ChannelFutureListener; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelInboundHandlerAdapter; +import io.netty.util.ReferenceCountUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.nio.charset.StandardCharsets; + /** * Handle data from client. * */ public class HttpProxyClientHandler extends ChannelInboundHandlerAdapter { + + private static final ByteBuf TUNNEL_OK = + Unpooled.unreleasableBuffer( + Unpooled.copiedBuffer("HTTP/1.1 200 Connection Established\r\n\r\n", StandardCharsets.US_ASCII)); + private final Logger logger = LoggerFactory.getLogger(HttpProxyClientHandler.class); private final String id; private Channel clientChannel; @@ -36,15 +44,27 @@ public void channelActive(ChannelHandlerContext ctx) { @Override public void channelRead(ChannelHandlerContext ctx, Object msg) { if (header.isComplete()) { - remoteChannel.writeAndFlush(msg); // just forward + if (remoteChannel != null && remoteChannel.isActive()) { + remoteChannel.writeAndFlush(msg); // just forward + } else { + ReferenceCountUtil.safeRelease(msg); + flushAndClose(clientChannel); + } return; } ByteBuf in = (ByteBuf) msg; - header.digest(in); + + try { + header.digest(in); + } catch (Throwable t) { + ReferenceCountUtil.safeRelease(in); + flushAndClose(clientChannel); + throw t; + } if (!header.isComplete()) { - in.release(); + ReferenceCountUtil.safeRelease(in); return; } @@ -52,7 +72,7 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) { clientChannel.config().setAutoRead(false); // disable AutoRead until remote connection is ready if (header.isHttps()) { // if https, respond 200 to create tunnel - clientChannel.writeAndFlush(Unpooled.wrappedBuffer("HTTP/1.1 200 Connection Established\r\n\r\n".getBytes())); + clientChannel.writeAndFlush(TUNNEL_OK.duplicate()); } Bootstrap b = new Bootstrap(); @@ -71,20 +91,27 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) { remoteChannel.writeAndFlush(in); } else { - in.release(); - clientChannel.close(); + ReferenceCountUtil.safeRelease(in); + // release header buffer if retained/allocated + ReferenceCountUtil.safeRelease(header.getByteBuf()); + flushAndClose(remoteChannel); + flushAndClose(clientChannel); } }); } @Override public void channelInactive(ChannelHandlerContext ctx) { + flushAndClose(remoteChannel); + remoteChannel = null; + clientChannel = null; } @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable e) { - logger.error(id + " error occured", e); + logger.error(id + " error occurred", e); + flushAndClose(remoteChannel); flushAndClose(clientChannel); } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/proxy/HttpProxyClientHeader.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/proxy/HttpProxyClientHeader.java index d300cbaa5337..4a64410fb187 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/proxy/HttpProxyClientHeader.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/proxy/HttpProxyClientHeader.java @@ -4,12 +4,20 @@ import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; +import io.netty.util.ResourceLeakDetector; /** * The http header of client. * */ public class HttpProxyClientHeader { + private static final int MAX_HEADER_BYTES = 64 * 1024; + + private static final boolean LEAK_DEBUG = + ResourceLeakDetector.getLevel().ordinal() >= ResourceLeakDetector.Level.ADVANCED.ordinal(); + + private volatile boolean touchedAlloc; + private String method; private String host; private int port; @@ -43,10 +51,6 @@ public int getPort() { return port; } - public void setPort(int port) { - this.port = port; - } - public boolean isHttps() { return https; } @@ -56,58 +60,87 @@ public void setHttps(boolean https) { } public ByteBuf getByteBuf() { - return byteBuf; - } - - public void setByteBuf(ByteBuf byteBuf) { - this.byteBuf = byteBuf; - } - - public StringBuilder getLineBuf() { - return lineBuf; + if (!complete) throw new IllegalStateException("header not complete"); + if (LEAK_DEBUG) { + this.byteBuf.touch("getByteBuf handoff"); + } + ByteBuf out = this.byteBuf; + this.byteBuf = Unpooled.EMPTY_BUFFER; // we no longer own anything + return out; // caller must write/release this } public void setComplete(boolean complete) { + if (complete) { + if (LEAK_DEBUG) { + this.byteBuf.touch("end-of-headers; https=" + https + " host=" + host + " port=" + port); + } + } this.complete = complete; } + /** Release any internal buffer we still own (idempotent). */ + public void releaseQuietly() { + if (LEAK_DEBUG) { + this.byteBuf.touch("releaseQuietly"); + } + io.netty.util.ReferenceCountUtil.safeRelease(this.byteBuf); + this.byteBuf = Unpooled.EMPTY_BUFFER; + } + public void digest(ByteBuf in) { + if (LEAK_DEBUG && !touchedAlloc) { + touchedAlloc = true; + this.byteBuf.touch("allocated header buffer"); + } + while (in.isReadable()) { - if (complete) { - throw new IllegalStateException("already complete"); - } + if (complete) throw new IllegalStateException("already complete"); String line = readLine(in); - if (line == null) { - return; - } + if (line == null) return; if (method == null) { - method = line.split(" ")[0]; // the first word is http method name - https = method.equalsIgnoreCase("CONNECT"); // method CONNECT means https + method = line.split(" ", 2)[0]; + https = "CONNECT".equalsIgnoreCase(method); } - if (line.startsWith("Host: ") || line.startsWith("host: ")) { - String[] arr = line.split(":"); - host = arr[1].trim(); - if (arr.length == 3) { - port = Integer.parseInt(arr[2]); - } else if (https) { - port = 443; // https + if (line.regionMatches(true, 0, "Host:", 0, 5)) { + // be tolerant to extra spaces and IPv6 + String value = line.substring(5).trim(); + int idx = value.lastIndexOf(':'); // last colon to allow IPv6 literals + if (idx > 0 && value.indexOf(']') < idx) { + host = value.substring(0, idx).trim(); + port = Integer.parseInt(value.substring(idx + 1).trim()); } else { - port = 80; // http + host = value; + port = https ? 443 : 80; } } if (line.isEmpty()) { if (host == null || port == 0) { - throw new IllegalStateException("cannot find header \'Host\'"); + releaseQuietly(); + throw new IllegalStateException("cannot find header 'Host'"); } - - byteBuf = byteBuf.asReadOnly(); - complete = true; + // If HTTPS, we don’t forward the CONNECT request → release now. + if (https) { + releaseQuietly(); + } else { + // non-HTTPS: make read-only for safety; caller will drain & own it + byteBuf = byteBuf.asReadOnly(); + } + setComplete(true); break; } + + // size guard to avoid OOM from giant headers + if (byteBuf.writerIndex() >= MAX_HEADER_BYTES) { + releaseQuietly(); + if (LEAK_DEBUG) { + this.byteBuf.touch("header too large at writerIndex=" + byteBuf.writerIndex()); + } + throw new IllegalStateException("header too large"); + } } } @@ -117,12 +150,14 @@ private String readLine(ByteBuf in) { byteBuf.writeByte(b); lineBuf.append((char) b); int len = lineBuf.length(); - if (len >= 2 && lineBuf.substring(len - 2).equals("\r\n")) { + if (len >= 2 && lineBuf.charAt(len - 2) == '\r' && lineBuf.charAt(len - 1) == '\n') { String line = lineBuf.substring(0, len - 2); - lineBuf.delete(0, len); + lineBuf.setLength(0); + if (LEAK_DEBUG) { + this.byteBuf.touch("CRLF reached, writerIndex=" + byteBuf.writerIndex()); + } return line; } - } return null; } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/proxy/HttpProxyRemoteHandler.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/proxy/HttpProxyRemoteHandler.java index 487aa265d104..ee9228aeeeb9 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/proxy/HttpProxyRemoteHandler.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/proxy/HttpProxyRemoteHandler.java @@ -32,18 +32,41 @@ public void channelActive(ChannelHandlerContext ctx) { @Override public void channelRead(ChannelHandlerContext ctx, Object msg) { - clientChannel.writeAndFlush(msg); // just forward + if (clientChannel != null && clientChannel.isActive()) { + clientChannel.writeAndFlush(msg).addListener(f -> { + if (!f.isSuccess()) { + // transport is broken; close both sides + flushAndClose(clientChannel); + flushAndClose(remoteChannel); + } + }); + } else { + io.netty.util.ReferenceCountUtil.safeRelease(msg); // prevent leak + flushAndClose(remoteChannel); + } } @Override public void channelInactive(ChannelHandlerContext ctx) { flushAndClose(clientChannel); + remoteChannel = null; + clientChannel = null; } @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable e) { - logger.error(id + " error occured", e); + logger.error(id + " error occurred", e); flushAndClose(remoteChannel); + flushAndClose(clientChannel); + } + + @Override + public void channelWritabilityChanged(ChannelHandlerContext ctx) throws Exception { + if (remoteChannel != null) { + boolean writable = clientChannel != null && clientChannel.isWritable(); + remoteChannel.config().setAutoRead(writable); + } + ctx.fireChannelWritabilityChanged(); } private void flushAndClose(Channel ch) { From 117a4e228b3ff4fb83641c73619b356e739d0266 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Thu, 13 Nov 2025 02:43:45 +0000 Subject: [PATCH 43/65] Test fixes --- .../CosmosNettyLeakDetectorFactory.java | 2 +- .../CosmosNettyLeakDetectorFactory.java | 2 +- .../ConnectionStateListenerTest.java | 147 +++++++++--------- 3 files changed, 77 insertions(+), 74 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java index 279b3b671323..182c6e5b8370 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java @@ -238,7 +238,7 @@ public static List resetIdentifiedLeaks() { public static AutoCloseable createDisableLeakDetectionScope() { synchronized (staticLock) { - logger.info("Disabling Leak detection: {}", StackTraceUtil.currentCallStack()); + logger.warn("Disabling Leak detection:"); isLeakDetectionDisabled = true; return new DisableLeakDetectionScope(); diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java index 919c0f2c5391..14a024b28a28 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java @@ -238,7 +238,7 @@ public static List resetIdentifiedLeaks() { public static AutoCloseable createDisableLeakDetectionScope() { synchronized (staticLock) { - logger.info("Disabling Leak detection: {}", StackTraceUtil.currentCallStack()); + logger.warn("Disabling Leak detection:"); isLeakDetectionDisabled = true; return new DisableLeakDetectionScope(); diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/ConnectionStateListenerTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/ConnectionStateListenerTest.java index 97f86bb795d3..4a994e86da47 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/ConnectionStateListenerTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/ConnectionStateListenerTest.java @@ -4,6 +4,7 @@ package com.azure.cosmos.implementation.directconnectivity; import com.azure.cosmos.ConnectionMode; +import com.azure.cosmos.CosmosNettyLeakDetectorFactory; import com.azure.cosmos.DirectConnectionConfig; import com.azure.cosmos.implementation.Configs; import com.azure.cosmos.implementation.ConnectionPolicy; @@ -88,82 +89,84 @@ public void connectionStateListener_OnConnectionEvent( boolean isTcpConnectionEndpointRediscoveryEnabled, RequestResponseType responseType, boolean markUnhealthy, - boolean markUnhealthyWhenServerShutdown) throws ExecutionException, InterruptedException, URISyntaxException { - - ConnectionPolicy connectionPolicy = new ConnectionPolicy(DirectConnectionConfig.getDefaultConfig()); - connectionPolicy.setTcpConnectionEndpointRediscoveryEnabled(isTcpConnectionEndpointRediscoveryEnabled); - - GlobalAddressResolver addressResolver = Mockito.mock(GlobalAddressResolver.class); - - SslContext sslContext = SslContextUtils.CreateSslContext("client.jks", false); - - Configs config = Mockito.mock(Configs.class); - Mockito.doReturn(sslContext).when(config).getSslContext(false, false); - - ClientTelemetry clientTelemetry = Mockito.mock(ClientTelemetry.class); - ClientTelemetryInfo clientTelemetryInfo = new ClientTelemetryInfo( - "testMachine", - "testClient", - "testProcess", - "testApp", - ConnectionMode.DIRECT, - "test-cdb-account", - "Test Region 1", - "Linux", - false, - Arrays.asList("Test Region 1", "Test Region 2")); - - Mockito.when(clientTelemetry.getClientTelemetryInfo()).thenReturn(clientTelemetryInfo); - - RntbdTransportClient client = new RntbdTransportClient( - config, - connectionPolicy, - new UserAgentContainer(), - addressResolver, - clientTelemetry, - null); - - RxDocumentServiceRequest req = - RxDocumentServiceRequest.create(mockDiagnosticsClientContext(), OperationType.Create, ResourceType.Document, - "dbs/fakedb/colls/fakeColls", - getDocumentDefinition(), new HashMap<>()); - req.requestContext.regionalRoutingContextToRoute = new RegionalRoutingContext(new URI("https://localhost:8080")); - - req.setPartitionKeyRangeIdentity(new PartitionKeyRangeIdentity("fakeCollectionId","fakePartitionKeyRangeId")); - - // Validate connectionStateListener will always track the latest Uri - List targetUris = new ArrayList<>(); - int serverPort = port + randomPort.getAndIncrement(); - String serverAddress = serverAddressPrefix + serverPort; - - targetUris.add(new Uri(serverAddress)); - targetUris.add(new Uri(serverAddress)); - - for (Uri uri : targetUris) { - // using a random generated server port - TcpServer server = TcpServerFactory.startNewRntbdServer(serverPort); - // Inject fake response - server.injectServerResponse(responseType); - - try { - client.invokeResourceOperationAsync(uri, req).block(); - } catch (Exception e) { - // no op here - } - - if (markUnhealthy) { - assertThat(uri.getHealthStatus()).isEqualTo(Uri.HealthStatus.Unhealthy); - TcpServerFactory.shutdownRntbdServer(server); - assertThat(uri.getHealthStatus()).isEqualTo(Uri.HealthStatus.Unhealthy); - - } else { - assertThat(uri.getHealthStatus()).isEqualTo(Uri.HealthStatus.Connected); + boolean markUnhealthyWhenServerShutdown) throws Exception { + + try (AutoCloseable disableNettyLeakDetectionScope = CosmosNettyLeakDetectorFactory.createDisableLeakDetectionScope()) { + ConnectionPolicy connectionPolicy = new ConnectionPolicy(DirectConnectionConfig.getDefaultConfig()); + connectionPolicy.setTcpConnectionEndpointRediscoveryEnabled(isTcpConnectionEndpointRediscoveryEnabled); + + GlobalAddressResolver addressResolver = Mockito.mock(GlobalAddressResolver.class); + + SslContext sslContext = SslContextUtils.CreateSslContext("client.jks", false); + + Configs config = Mockito.mock(Configs.class); + Mockito.doReturn(sslContext).when(config).getSslContext(false, false); + + ClientTelemetry clientTelemetry = Mockito.mock(ClientTelemetry.class); + ClientTelemetryInfo clientTelemetryInfo = new ClientTelemetryInfo( + "testMachine", + "testClient", + "testProcess", + "testApp", + ConnectionMode.DIRECT, + "test-cdb-account", + "Test Region 1", + "Linux", + false, + Arrays.asList("Test Region 1", "Test Region 2")); + + Mockito.when(clientTelemetry.getClientTelemetryInfo()).thenReturn(clientTelemetryInfo); + + RntbdTransportClient client = new RntbdTransportClient( + config, + connectionPolicy, + new UserAgentContainer(), + addressResolver, + clientTelemetry, + null); + + RxDocumentServiceRequest req = + RxDocumentServiceRequest.create(mockDiagnosticsClientContext(), OperationType.Create, ResourceType.Document, + "dbs/fakedb/colls/fakeColls", + getDocumentDefinition(), new HashMap<>()); + req.requestContext.regionalRoutingContextToRoute = new RegionalRoutingContext(new URI("https://localhost:8080")); + + req.setPartitionKeyRangeIdentity(new PartitionKeyRangeIdentity("fakeCollectionId", "fakePartitionKeyRangeId")); + + // Validate connectionStateListener will always track the latest Uri + List targetUris = new ArrayList<>(); + int serverPort = port + randomPort.getAndIncrement(); + String serverAddress = serverAddressPrefix + serverPort; + + targetUris.add(new Uri(serverAddress)); + targetUris.add(new Uri(serverAddress)); + + for (Uri uri : targetUris) { + // using a random generated server port + TcpServer server = TcpServerFactory.startNewRntbdServer(serverPort); + // Inject fake response + server.injectServerResponse(responseType); + + try { + client.invokeResourceOperationAsync(uri, req).block(); + } catch (Exception e) { + // no op here + } - TcpServerFactory.shutdownRntbdServer(server); - if (markUnhealthyWhenServerShutdown) { + if (markUnhealthy) { assertThat(uri.getHealthStatus()).isEqualTo(Uri.HealthStatus.Unhealthy); + TcpServerFactory.shutdownRntbdServer(server); + assertThat(uri.getHealthStatus()).isEqualTo(Uri.HealthStatus.Unhealthy); + } else { assertThat(uri.getHealthStatus()).isEqualTo(Uri.HealthStatus.Connected); + + TcpServerFactory.shutdownRntbdServer(server); + if (markUnhealthyWhenServerShutdown) { + assertThat(uri.getHealthStatus()).isEqualTo(Uri.HealthStatus.Unhealthy); + } else { + assertThat(uri.getHealthStatus()).isEqualTo(Uri.HealthStatus.Connected); + } } } } From 7a30b0e6689a065a47142fea4feead6ee28cce29 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Thu, 13 Nov 2025 03:06:29 +0000 Subject: [PATCH 44/65] Users/fabianm/portfixes (#47252) * Update WebExceptionRetryPolicy.java * Update ThinClientStoreModel.java --- .../cosmos/implementation/ThinClientStoreModel.java | 10 +++++++++- .../cosmos/implementation/WebExceptionRetryPolicy.java | 4 ++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ThinClientStoreModel.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ThinClientStoreModel.java index 1bd343e6192b..2a92039ddd7e 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ThinClientStoreModel.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ThinClientStoreModel.java @@ -135,13 +135,21 @@ public StoreResponse unwrapToStoreResponse( payloadBuf ); + if (payloadBuf != content) { + // payload is a slice/derived view; super() owns payload, we still own the container + // this includes scenarios where payloadBuf == EMPTY_BUFFER + ReferenceCountUtil.safeRelease(content); + } + return storeResponse; - } finally { + } catch (Throwable t){ if (payloadBuf != content) { // payload is a slice/derived view; super() owns payload, we still own the container // this includes scenarios where payloadBuf == EMPTY_BUFFER ReferenceCountUtil.safeRelease(content); } + + throw t; } } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/WebExceptionRetryPolicy.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/WebExceptionRetryPolicy.java index b1f737530f31..0314d4dea86c 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/WebExceptionRetryPolicy.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/WebExceptionRetryPolicy.java @@ -48,9 +48,9 @@ public Mono shouldRetry(Exception e) { .warn( "WebExceptionRetryPolicy() No retries because client is not initialized yet. - " + "operationType = {}, count = {}, isAddressRefresh = {}", - this.request.getOperationType(), + this.request != null ? this.request.getOperationType() : "n/a", this.retryCount, - this.request.isAddressRefresh()); + this.request != null ? this.request.isAddressRefresh() : "n/a"); return Mono.just(ShouldRetryResult.noRetry()); } From 4b97efff1a433566162acea1a21dadc944b84292 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Thu, 13 Nov 2025 13:38:47 +0000 Subject: [PATCH 45/65] Test fixes --- .../CosmosNettyLeakDetectorFactory.java | 27 +- .../azure/cosmos/CosmosClientBuilderTest.java | 255 +++++++++--------- .../CosmosNettyLeakDetectorFactory.java | 27 +- 3 files changed, 162 insertions(+), 147 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java index 182c6e5b8370..f19ef115fa4f 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java @@ -4,7 +4,6 @@ import com.azure.cosmos.implementation.Configs; import com.azure.cosmos.implementation.RxDocumentClientImpl; -import com.azure.cosmos.implementation.StackTraceUtil; import io.netty.buffer.PooledByteBufAllocator; import io.netty.util.ResourceLeakDetector; import io.netty.util.ResourceLeakDetectorFactory; @@ -16,12 +15,14 @@ import org.testng.IInvokedMethod; import org.testng.IInvokedMethodListener; import org.testng.ITestClass; +import org.testng.ITestContext; import org.testng.ITestNGMethod; import org.testng.ITestResult; import java.lang.management.BufferPoolMXBean; import java.lang.management.ManagementFactory; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -84,26 +85,28 @@ public void onAfterClass(ITestClass testClass) { // -Dtestng.listener.execution.symmetric=true allows, but this is only available // in TestNG 7.7.1 - which requires Java11 // So, this class simulates this behavior by hooking into IInvokedMethodListener - // If the test class itself does not have @afterClass we execute the logic here - - ITestNGMethod[] afterClassMethods = testClass.getAfterClassMethods(); - boolean testClassHasAfterClassMethods = afterClassMethods != null && afterClassMethods.length > 0; - if (!testClassHasAfterClassMethods) { - this.onAfterClassCore(testClass); - } } @Override - public void afterInvocation(IInvokedMethod method, ITestResult result) { + public void afterInvocation(IInvokedMethod method, ITestResult result, ITestContext ctx) { ITestClass testClass = (ITestClass)result.getTestClass(); ITestNGMethod[] afterClassMethods = testClass.getAfterClassMethods(); boolean testClassHasAfterClassMethods = afterClassMethods != null && afterClassMethods.length > 0; - if (testClassHasAfterClassMethods + boolean isImplementedAfterClassMethod = testClassHasAfterClassMethods && method.isConfigurationMethod() - && method.getTestMethod().isAfterClassConfiguration()) { + && method.getTestMethod().isAfterClassConfiguration(); + + ITestNGMethod[] testMethods = ctx.getAllTestMethods(); + + boolean isLastTestMethodOnTestClassWithoutAfterClassMethod = !testClassHasAfterClassMethods + && method.isTestMethod() + && method.getTestMethod().isTest() + && method.getTestMethod().getEnabled() + && testMethods.length > 0 + && method.getTestMethod() == testMethods[testMethods.length - 1]; - // <-- This point is guaranteed to be AFTER the class’s @AfterClass ran if any existed + if (isImplementedAfterClassMethod || isLastTestMethodOnTestClassWithoutAfterClassMethod) { this.onAfterClassCore(testClass); } } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosClientBuilderTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosClientBuilderTest.java index 53c03033ee3c..79121376b871 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosClientBuilderTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosClientBuilderTest.java @@ -61,12 +61,11 @@ public void validateBadPreferredRegions1() { @Test(groups = "unit") public void validateBadPreferredRegions2() { - try { - CosmosAsyncClient client = new CosmosClientBuilder() + try (CosmosAsyncClient client = new CosmosClientBuilder() .key(TestConfigurations.MASTER_KEY) .endpoint(hostName) .preferredRegions(Arrays.asList(" ")) - .buildAsyncClient(); + .buildAsyncClient()) { client.close(); } catch (Exception e) { assertThat(e).isInstanceOf(RuntimeException.class); @@ -92,9 +91,11 @@ public void validateApiTypePresent() { ImplementationBridgeHelpers.CosmosClientBuilderHelper.getCosmosClientBuilderAccessor(); accessor.setCosmosClientApiType(cosmosClientBuilder, apiType); - RxDocumentClientImpl documentClient = - (RxDocumentClientImpl) ReflectionUtils.getAsyncDocumentClient(cosmosClientBuilder.buildAsyncClient()); - assertThat(ReflectionUtils.getApiType(documentClient)).isEqualTo(apiType); + try (CosmosAsyncClient cosmosClient = cosmosClientBuilder.buildAsyncClient()) { + RxDocumentClientImpl documentClient = + (RxDocumentClientImpl) ReflectionUtils.getAsyncDocumentClient(cosmosClient); + assertThat(ReflectionUtils.getApiType(documentClient)).isEqualTo(apiType); + } } @Test(groups = "emulator", dataProvider = "regionScopedSessionContainerConfigs") @@ -111,23 +112,24 @@ public void validateSessionTokenCapturingForAccountDefaultConsistency(boolean sh .key(TestConfigurations.MASTER_KEY) .userAgentSuffix("custom-direct-client"); - CosmosAsyncClient client = cosmosClientBuilder.buildAsyncClient(); - RxDocumentClientImpl documentClient = - (RxDocumentClientImpl) ReflectionUtils.getAsyncDocumentClient(client); + try (CosmosAsyncClient client = cosmosClientBuilder.buildAsyncClient()) { + RxDocumentClientImpl documentClient = + (RxDocumentClientImpl) ReflectionUtils.getAsyncDocumentClient(client); - if (documentClient.getDefaultConsistencyLevelOfAccount() != ConsistencyLevel.SESSION) { - throw new SkipException("This test is only applicable when default account-level consistency is Session."); - } + if (documentClient.getDefaultConsistencyLevelOfAccount() != ConsistencyLevel.SESSION) { + throw new SkipException("This test is only applicable when default account-level consistency is Session."); + } - ISessionContainer sessionContainer = documentClient.getSession(); + ISessionContainer sessionContainer = documentClient.getSession(); - if (System.getProperty("COSMOS.SESSION_CAPTURING_TYPE") != null && System.getProperty("COSMOS.SESSION_CAPTURING_TYPE").equals("REGION_SCOPED")) { - assertThat(sessionContainer instanceof RegionScopedSessionContainer).isTrue(); - } else { - assertThat(sessionContainer instanceof SessionContainer).isTrue(); - } + if (System.getProperty("COSMOS.SESSION_CAPTURING_TYPE") != null && System.getProperty("COSMOS.SESSION_CAPTURING_TYPE").equals("REGION_SCOPED")) { + assertThat(sessionContainer instanceof RegionScopedSessionContainer).isTrue(); + } else { + assertThat(sessionContainer instanceof SessionContainer).isTrue(); + } - assertThat(sessionContainer.getDisableSessionCapturing()).isEqualTo(false); + assertThat(sessionContainer.getDisableSessionCapturing()).isEqualTo(false); + } } finally { System.clearProperty("COSMOS.SESSION_CAPTURING_TYPE"); } @@ -144,23 +146,24 @@ public void validateSessionTokenCapturingForAccountDefaultConsistencyWithEnvVari .key(TestConfigurations.MASTER_KEY) .userAgentSuffix("custom-direct-client"); - CosmosAsyncClient client = cosmosClientBuilder.buildAsyncClient(); - RxDocumentClientImpl documentClient = - (RxDocumentClientImpl) ReflectionUtils.getAsyncDocumentClient(client); + try (CosmosAsyncClient client = cosmosClientBuilder.buildAsyncClient()) { + RxDocumentClientImpl documentClient = + (RxDocumentClientImpl) ReflectionUtils.getAsyncDocumentClient(client); - if (documentClient.getDefaultConsistencyLevelOfAccount() != ConsistencyLevel.SESSION) { - throw new SkipException("This test is only applicable when default account-level consistency is Session."); - } + if (documentClient.getDefaultConsistencyLevelOfAccount() != ConsistencyLevel.SESSION) { + throw new SkipException("This test is only applicable when default account-level consistency is Session."); + } - ISessionContainer sessionContainer = documentClient.getSession(); + ISessionContainer sessionContainer = documentClient.getSession(); - if (System.getenv("COSMOS.SESSION_CAPTURING_TYPE") != null && System.getenv("COSMOS.SESSION_CAPTURING_TYPE").equals("REGION_SCOPED")) { - assertThat(sessionContainer instanceof RegionScopedSessionContainer).isTrue(); - } else { - assertThat(sessionContainer instanceof SessionContainer).isTrue(); - } + if (System.getenv("COSMOS.SESSION_CAPTURING_TYPE") != null && System.getenv("COSMOS.SESSION_CAPTURING_TYPE").equals("REGION_SCOPED")) { + assertThat(sessionContainer instanceof RegionScopedSessionContainer).isTrue(); + } else { + assertThat(sessionContainer instanceof SessionContainer).isTrue(); + } - assertThat(sessionContainer.getDisableSessionCapturing()).isEqualTo(false); + assertThat(sessionContainer.getDisableSessionCapturing()).isEqualTo(false); + } } finally { System.clearProperty("COSMOS.SESSION_CAPTURING_TYPE"); } @@ -168,102 +171,108 @@ public void validateSessionTokenCapturingForAccountDefaultConsistencyWithEnvVari @Test(groups = "emulator") public void validateContainerCreationInterceptor() { - CosmosClient clientWithoutInterceptor = new CosmosClientBuilder() + try (CosmosClient clientWithoutInterceptor = new CosmosClientBuilder() .endpoint(TestConfigurations.HOST) .key(TestConfigurations.MASTER_KEY) .userAgentSuffix("noInterceptor") - .buildClient(); - - ConcurrentMap> queryCache = new ConcurrentHashMap<>(); - - CosmosClient clientWithInterceptor = new CosmosClientBuilder() - .endpoint(TestConfigurations.HOST) - .key(TestConfigurations.MASTER_KEY) - .userAgentSuffix("withInterceptor") - .containerCreationInterceptor(originalContainer -> new CacheAndValidateQueriesContainer(originalContainer, queryCache)) - .buildClient(); - - CosmosAsyncClient asyncClientWithInterceptor = new CosmosClientBuilder() - .endpoint(TestConfigurations.HOST) - .key(TestConfigurations.MASTER_KEY) - .userAgentSuffix("withInterceptor") - .containerCreationInterceptor(originalContainer -> new CacheAndValidateQueriesContainer(originalContainer, queryCache)) - .buildAsyncClient(); - - CosmosContainer normalContainer = clientWithoutInterceptor - .getDatabase("TestDB") - .getContainer("TestContainer"); - assertThat(normalContainer).isNotNull(); - assertThat(normalContainer.getClass()).isEqualTo(CosmosContainer.class); - assertThat(normalContainer.asyncContainer.getClass()).isEqualTo(CosmosAsyncContainer.class); - - CosmosContainer customSyncContainer = clientWithInterceptor - .getDatabase("TestDB") - .getContainer("TestContainer"); - assertThat(customSyncContainer).isNotNull(); - assertThat(customSyncContainer.getClass()).isEqualTo(CosmosContainer.class); - assertThat(customSyncContainer.asyncContainer.getClass()).isEqualTo(CacheAndValidateQueriesContainer.class); - - CosmosAsyncContainer customAsyncContainer = asyncClientWithInterceptor - .getDatabase("TestDB") - .getContainer("TestContainer"); - assertThat(customAsyncContainer).isNotNull(); - assertThat(customAsyncContainer.getClass()).isEqualTo(CacheAndValidateQueriesContainer.class); - - try { - customSyncContainer.queryItems("SELECT * from c", null, ObjectNode.class); - fail("Unparameterized query should throw"); - } catch (IllegalStateException expectedError) {} - - try { - customAsyncContainer.queryItems("SELECT * from c", null, ObjectNode.class); - fail("Unparameterized query should throw"); - } catch (IllegalStateException expectedError) {} - - try { - customAsyncContainer.queryItems("SELECT * from c", ObjectNode.class); - fail("Unparameterized query should throw"); - } catch (IllegalStateException expectedError) {} - - SqlQuerySpec querySpec = new SqlQuerySpec().setQueryText("SELECT * from c"); - assertThat(queryCache).size().isEqualTo(0); - - try { - List items = customSyncContainer - .queryItems(querySpec, null, ObjectNode.class) - .stream().collect(Collectors.toList()); - fail("Not yet cached - the query above should always throw"); - } catch (CosmosException cosmosException) { - // Container does not exist - when not cached should fail - assertThat(cosmosException.getStatusCode()).isEqualTo(404); - assertThat(cosmosException.getSubStatusCode()).isEqualTo(1003); - } - - queryCache.putIfAbsent(new CacheKey(ObjectNode.class.getCanonicalName(), querySpec), new ArrayList<>()); - assertThat(queryCache).size().isEqualTo(1); - - // Validate that CacheKey equality check works - queryCache.putIfAbsent(new CacheKey(ObjectNode.class.getCanonicalName(), querySpec), new ArrayList<>()); - assertThat(queryCache).size().isEqualTo(1); - - // Validate that form cache the results can be served - List items = customSyncContainer - .queryItems(querySpec, null, ObjectNode.class) - .stream().collect(Collectors.toList()); + .buildClient()) { - querySpec = new SqlQuerySpec().setQueryText("SELECT * from c"); - CosmosPagedFlux cachedPagedFlux = customAsyncContainer - .queryItems(querySpec, null, ObjectNode.class); - assertThat(cachedPagedFlux.getClass().getName()).startsWith("com.azure.cosmos.util.CosmosPagedFluxStaticListImpl"); + ConcurrentMap> queryCache = new ConcurrentHashMap<>(); - // Validate that uncached query form async Container also fails with 404 due to non-existing Container - querySpec = new SqlQuerySpec().setQueryText("SELECT * from r"); - try { - CosmosPagedFlux uncachedPagedFlux = customAsyncContainer - .queryItems(querySpec, null, ObjectNode.class); - } catch (CosmosException cosmosException) { - assertThat(cosmosException.getStatusCode()).isEqualTo(404); - assertThat(cosmosException.getSubStatusCode()).isEqualTo(1003); + try (CosmosClient clientWithInterceptor = new CosmosClientBuilder() + .endpoint(TestConfigurations.HOST) + .key(TestConfigurations.MASTER_KEY) + .userAgentSuffix("withInterceptor") + .containerCreationInterceptor(originalContainer -> new CacheAndValidateQueriesContainer(originalContainer, queryCache)) + .buildClient()) { + + try (CosmosAsyncClient asyncClientWithInterceptor = new CosmosClientBuilder() + .endpoint(TestConfigurations.HOST) + .key(TestConfigurations.MASTER_KEY) + .userAgentSuffix("withInterceptor") + .containerCreationInterceptor(originalContainer -> new CacheAndValidateQueriesContainer(originalContainer, queryCache)) + .buildAsyncClient()) { + + CosmosContainer normalContainer = clientWithoutInterceptor + .getDatabase("TestDB") + .getContainer("TestContainer"); + assertThat(normalContainer).isNotNull(); + assertThat(normalContainer.getClass()).isEqualTo(CosmosContainer.class); + assertThat(normalContainer.asyncContainer.getClass()).isEqualTo(CosmosAsyncContainer.class); + + CosmosContainer customSyncContainer = clientWithInterceptor + .getDatabase("TestDB") + .getContainer("TestContainer"); + assertThat(customSyncContainer).isNotNull(); + assertThat(customSyncContainer.getClass()).isEqualTo(CosmosContainer.class); + assertThat(customSyncContainer.asyncContainer.getClass()).isEqualTo(CacheAndValidateQueriesContainer.class); + + CosmosAsyncContainer customAsyncContainer = asyncClientWithInterceptor + .getDatabase("TestDB") + .getContainer("TestContainer"); + assertThat(customAsyncContainer).isNotNull(); + assertThat(customAsyncContainer.getClass()).isEqualTo(CacheAndValidateQueriesContainer.class); + + try { + customSyncContainer.queryItems("SELECT * from c", null, ObjectNode.class); + fail("Unparameterized query should throw"); + } catch (IllegalStateException expectedError) { + } + + try { + customAsyncContainer.queryItems("SELECT * from c", null, ObjectNode.class); + fail("Unparameterized query should throw"); + } catch (IllegalStateException expectedError) { + } + + try { + customAsyncContainer.queryItems("SELECT * from c", ObjectNode.class); + fail("Unparameterized query should throw"); + } catch (IllegalStateException expectedError) { + } + + SqlQuerySpec querySpec = new SqlQuerySpec().setQueryText("SELECT * from c"); + assertThat(queryCache).size().isEqualTo(0); + + try { + List items = customSyncContainer + .queryItems(querySpec, null, ObjectNode.class) + .stream().collect(Collectors.toList()); + fail("Not yet cached - the query above should always throw"); + } catch (CosmosException cosmosException) { + // Container does not exist - when not cached should fail + assertThat(cosmosException.getStatusCode()).isEqualTo(404); + assertThat(cosmosException.getSubStatusCode()).isEqualTo(1003); + } + + queryCache.putIfAbsent(new CacheKey(ObjectNode.class.getCanonicalName(), querySpec), new ArrayList<>()); + assertThat(queryCache).size().isEqualTo(1); + + // Validate that CacheKey equality check works + queryCache.putIfAbsent(new CacheKey(ObjectNode.class.getCanonicalName(), querySpec), new ArrayList<>()); + assertThat(queryCache).size().isEqualTo(1); + + // Validate that form cache the results can be served + List items = customSyncContainer + .queryItems(querySpec, null, ObjectNode.class) + .stream().collect(Collectors.toList()); + + querySpec = new SqlQuerySpec().setQueryText("SELECT * from c"); + CosmosPagedFlux cachedPagedFlux = customAsyncContainer + .queryItems(querySpec, null, ObjectNode.class); + assertThat(cachedPagedFlux.getClass().getName()).startsWith("com.azure.cosmos.util.CosmosPagedFluxStaticListImpl"); + + // Validate that uncached query form async Container also fails with 404 due to non-existing Container + querySpec = new SqlQuerySpec().setQueryText("SELECT * from r"); + try { + CosmosPagedFlux uncachedPagedFlux = customAsyncContainer + .queryItems(querySpec, null, ObjectNode.class); + } catch (CosmosException cosmosException) { + assertThat(cosmosException.getStatusCode()).isEqualTo(404); + assertThat(cosmosException.getSubStatusCode()).isEqualTo(1003); + } + } + } } } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java index 14a024b28a28..a4f4dd7e5878 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java @@ -4,7 +4,6 @@ import com.azure.cosmos.implementation.Configs; import com.azure.cosmos.implementation.RxDocumentClientImpl; -import com.azure.cosmos.implementation.StackTraceUtil; import io.netty.buffer.PooledByteBufAllocator; import io.netty.util.ResourceLeakDetector; import io.netty.util.ResourceLeakDetectorFactory; @@ -16,12 +15,14 @@ import org.testng.IInvokedMethod; import org.testng.IInvokedMethodListener; import org.testng.ITestClass; +import org.testng.ITestContext; import org.testng.ITestNGMethod; import org.testng.ITestResult; import java.lang.management.BufferPoolMXBean; import java.lang.management.ManagementFactory; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -84,26 +85,28 @@ public void onAfterClass(ITestClass testClass) { // -Dtestng.listener.execution.symmetric=true allows, but this is only available // in TestNG 7.7.1 - which requires Java11 // So, this class simulates this behavior by hooking into IInvokedMethodListener - // If the test class itself does not have @afterClass we execute the logic here - - ITestNGMethod[] afterClassMethods = testClass.getAfterClassMethods(); - boolean testClassHasAfterClassMethods = afterClassMethods != null && afterClassMethods.length > 0; - if (!testClassHasAfterClassMethods) { - this.onAfterClassCore(testClass); - } } @Override - public void afterInvocation(IInvokedMethod method, ITestResult result) { + public void afterInvocation(IInvokedMethod method, ITestResult result, ITestContext ctx) { ITestClass testClass = (ITestClass)result.getTestClass(); ITestNGMethod[] afterClassMethods = testClass.getAfterClassMethods(); boolean testClassHasAfterClassMethods = afterClassMethods != null && afterClassMethods.length > 0; - if (testClassHasAfterClassMethods + boolean isImplementedAfterClassMethod = testClassHasAfterClassMethods && method.isConfigurationMethod() - && method.getTestMethod().isAfterClassConfiguration()) { + && method.getTestMethod().isAfterClassConfiguration(); + + ITestNGMethod[] testMethods = ctx.getAllTestMethods(); + + boolean isLastTestMethodOnTestClassWithoutAfterClassMethod = !testClassHasAfterClassMethods + && method.isTestMethod() + && method.getTestMethod().isTest() + && method.getTestMethod().getEnabled() + && testMethods.length > 0 + && method.getTestMethod() == testMethods[testMethods.length - 1]; - // <-- This point is guaranteed to be AFTER the class’s @AfterClass ran if any existed + if (isImplementedAfterClassMethod || isLastTestMethodOnTestClassWithoutAfterClassMethod) { this.onAfterClassCore(testClass); } } From 06425ebc3309b13cdf6946da9010b671ddf1bbfb Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Thu, 13 Nov 2025 19:55:49 +0100 Subject: [PATCH 46/65] Fix Netty ByteBuf leaks in StoreResponse and RetryContextOnDiagnosticTest (#47266) * Initial plan * Improve logging for ByteBufInputStream close failures Change log level from debug to warn and catch Throwable instead of just IOException to make potential ByteBuf leak issues more visible. Co-authored-by: FabianMeiswinkel <19165014+FabianMeiswinkel@users.noreply.github.com> * Fix ByteBuf leak in RetryContextOnDiagnosticTest Changed from Mono.just() to Mono.fromCallable() to defer StoreResponse creation, ensuring ByteBuf lifecycle is properly managed within each subscription rather than eagerly at mock setup time. Co-authored-by: FabianMeiswinkel <19165014+FabianMeiswinkel@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: FabianMeiswinkel <19165014+FabianMeiswinkel@users.noreply.github.com> --- .../cosmos/RetryContextOnDiagnosticTest.java | 22 +++++++------------ .../directconnectivity/StoreResponse.java | 8 +++---- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/RetryContextOnDiagnosticTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/RetryContextOnDiagnosticTest.java index 7d6491c1fee2..f0d46402a0b4 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/RetryContextOnDiagnosticTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/RetryContextOnDiagnosticTest.java @@ -105,15 +105,12 @@ public void backoffRetryUtilityExecuteRetry() throws Exception { addressSelector = Mockito.mock(AddressSelector.class); CosmosException exception = new CosmosException(410, exceptionText); String rawJson = "{\"id\":\"" + responseText + "\"}"; - ByteBuf buffer = getUTF8BytesOrNull(rawJson); Mockito.when(callbackMethod.call()).thenThrow(exception, exception, exception, exception, exception) - .thenReturn(Mono.just(new StoreResponse( - null, - 200, - new HashMap<>(), - new ByteBufInputStream(buffer, true), - buffer.readableBytes()))); + .thenReturn(Mono.fromCallable(() -> StoreResponseBuilder.create() + .withContent(rawJson) + .withStatus(200) + .build())); Mono monoResponse = BackoffRetryUtility.executeRetry(callbackMethod, retryPolicy); StoreResponse response = validateSuccess(monoResponse); @@ -156,14 +153,11 @@ public void backoffRetryUtilityExecuteAsync() { CosmosException exception = new CosmosException(410, exceptionText); Mono exceptionMono = Mono.error(exception); String rawJson = "{\"id\":\"" + responseText + "\"}"; - ByteBuf buffer = getUTF8BytesOrNull(rawJson); Mockito.when(parameterizedCallbackMethod.apply(ArgumentMatchers.any())).thenReturn(exceptionMono, exceptionMono, exceptionMono, exceptionMono, exceptionMono) - .thenReturn(Mono.just(new StoreResponse( - null, - 200, - new HashMap<>(), - new ByteBufInputStream(buffer, true), - buffer.readableBytes()))); + .thenReturn(Mono.fromCallable(() -> StoreResponseBuilder.create() + .withContent(rawJson) + .withStatus(200) + .build())); Mono monoResponse = BackoffRetryUtility.executeAsync( parameterizedCallbackMethod, retryPolicy, diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreResponse.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreResponse.java index c8539f09cba5..a4a3cccd558c 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreResponse.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreResponse.java @@ -71,12 +71,12 @@ public StoreResponse( if (contentStream != null) { try { this.responsePayload = new JsonNodeStorePayload(contentStream, responsePayloadLength, headerMap); - } - finally { + } finally { try { contentStream.close(); - } catch (IOException e) { - logger.debug("Could not successfully close content stream.", e); + } catch (Throwable e) { + // Log as warning instead of debug to make ByteBuf leak issues more visible + logger.warn("Failed to close content stream. This may cause a Netty ByteBuf leak.", e); } } } else { From daa8fd3b977f929bfa9d1d66aec6fd1b9a2d5289 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Thu, 13 Nov 2025 19:07:47 +0000 Subject: [PATCH 47/65] Update EncryptionAsyncApiCrudTest.java --- .../EncryptionAsyncApiCrudTest.java | 209 +++++++++--------- 1 file changed, 105 insertions(+), 104 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/EncryptionAsyncApiCrudTest.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/EncryptionAsyncApiCrudTest.java index 1fbda8eb17f0..c9ee457fa750 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/EncryptionAsyncApiCrudTest.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/EncryptionAsyncApiCrudTest.java @@ -509,117 +509,118 @@ public void crudOnDifferentOverload() { String databaseId = UUID.randomUUID().toString(); try { createNewDatabaseWithClientEncryptionKey(databaseId); - CosmosAsyncClient asyncClient = getClientBuilder().buildAsyncClient(); - KeyEncryptionKeyResolver keyEncryptionKeyResolver = new TestKeyEncryptionKeyResolver(); - CosmosEncryptionAsyncClient cosmosEncryptionAsyncClient = new CosmosEncryptionClientBuilder().cosmosAsyncClient(asyncClient).keyEncryptionKeyResolver( - keyEncryptionKeyResolver).keyEncryptionKeyResolverName("TEST_KEY_RESOLVER").buildAsyncClient(); - CosmosEncryptionAsyncDatabase cosmosEncryptionAsyncDatabase = - cosmosEncryptionAsyncClient.getCosmosEncryptionAsyncDatabase(asyncClient.getDatabase(databaseId)); - - String containerId = UUID.randomUUID().toString(); - ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(getPaths(1), 1); - createEncryptionContainer(cosmosEncryptionAsyncDatabase, clientEncryptionPolicy, containerId); - CosmosEncryptionAsyncContainer encryptionAsyncContainerOriginal = - cosmosEncryptionAsyncDatabase.getCosmosEncryptionAsyncContainer(containerId); - - List actualProperties = new ArrayList<>(); - // Read item - EncryptionPojo properties = getItem(UUID.randomUUID().toString()); - CosmosItemResponse itemResponse = encryptionAsyncContainerOriginal.createItem(properties).block(); - assertThat(itemResponse.getRequestCharge()).isGreaterThan(0); - EncryptionPojo responseItem = itemResponse.getItem(); - validateResponse(properties, responseItem); - actualProperties.add(properties); - - properties = getItem(UUID.randomUUID().toString()); - CosmosItemResponse itemResponse1 = encryptionAsyncContainerOriginal.createItem(properties, new CosmosItemRequestOptions()).block(); - assertThat(itemResponse1.getRequestCharge()).isGreaterThan(0); - EncryptionPojo responseItem1 = itemResponse1.getItem(); - validateResponse(properties, responseItem1); - actualProperties.add(properties); - - //Upsert Item - properties = getItem(UUID.randomUUID().toString()); - CosmosItemResponse upsertResponse1 = encryptionAsyncContainerOriginal.upsertItem(properties).block(); - assertThat(upsertResponse1.getRequestCharge()).isGreaterThan(0); - EncryptionPojo responseItem2 = upsertResponse1.getItem(); - validateResponse(properties, responseItem2); - actualProperties.add(properties); - - properties = getItem(UUID.randomUUID().toString()); - CosmosItemResponse upsertResponse2 = encryptionAsyncContainerOriginal.upsertItem(properties, new CosmosItemRequestOptions()).block(); - assertThat(upsertResponse2.getRequestCharge()).isGreaterThan(0); - EncryptionPojo responseItem3 = upsertResponse2.getItem(); - validateResponse(properties, responseItem3); - actualProperties.add(properties); - - //Read Item - EncryptionPojo readItem = encryptionAsyncContainerOriginal.readItem(actualProperties.get(0).getId(), - new PartitionKey(actualProperties.get(0).getMypk()), EncryptionPojo.class).block().getItem(); - validateResponse(actualProperties.get(0), readItem); - - //Query Item - String query = String.format("SELECT * from c where c.id = '%s'", actualProperties.get(1).getId()); - - CosmosPagedFlux feedResponseIterator = - encryptionAsyncContainerOriginal.queryItems(query, EncryptionPojo.class); - List feedResponse = feedResponseIterator.byPage().blockFirst().getResults(); - assertThat(feedResponse.size()).isGreaterThanOrEqualTo(1); - for (EncryptionPojo pojo : feedResponse) { - if (pojo.getId().equals(actualProperties.get(1).getId())) { - validateResponse(pojo, responseItem1); + try (CosmosAsyncClient asyncClient = getClientBuilder().buildAsyncClient()) { + KeyEncryptionKeyResolver keyEncryptionKeyResolver = new TestKeyEncryptionKeyResolver(); + CosmosEncryptionAsyncClient cosmosEncryptionAsyncClient = new CosmosEncryptionClientBuilder().cosmosAsyncClient(asyncClient).keyEncryptionKeyResolver( + keyEncryptionKeyResolver).keyEncryptionKeyResolverName("TEST_KEY_RESOLVER").buildAsyncClient(); + CosmosEncryptionAsyncDatabase cosmosEncryptionAsyncDatabase = + cosmosEncryptionAsyncClient.getCosmosEncryptionAsyncDatabase(asyncClient.getDatabase(databaseId)); + + String containerId = UUID.randomUUID().toString(); + ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(getPaths(1), 1); + createEncryptionContainer(cosmosEncryptionAsyncDatabase, clientEncryptionPolicy, containerId); + CosmosEncryptionAsyncContainer encryptionAsyncContainerOriginal = + cosmosEncryptionAsyncDatabase.getCosmosEncryptionAsyncContainer(containerId); + + List actualProperties = new ArrayList<>(); + // Read item + EncryptionPojo properties = getItem(UUID.randomUUID().toString()); + CosmosItemResponse itemResponse = encryptionAsyncContainerOriginal.createItem(properties).block(); + assertThat(itemResponse.getRequestCharge()).isGreaterThan(0); + EncryptionPojo responseItem = itemResponse.getItem(); + validateResponse(properties, responseItem); + actualProperties.add(properties); + + properties = getItem(UUID.randomUUID().toString()); + CosmosItemResponse itemResponse1 = encryptionAsyncContainerOriginal.createItem(properties, new CosmosItemRequestOptions()).block(); + assertThat(itemResponse1.getRequestCharge()).isGreaterThan(0); + EncryptionPojo responseItem1 = itemResponse1.getItem(); + validateResponse(properties, responseItem1); + actualProperties.add(properties); + + //Upsert Item + properties = getItem(UUID.randomUUID().toString()); + CosmosItemResponse upsertResponse1 = encryptionAsyncContainerOriginal.upsertItem(properties).block(); + assertThat(upsertResponse1.getRequestCharge()).isGreaterThan(0); + EncryptionPojo responseItem2 = upsertResponse1.getItem(); + validateResponse(properties, responseItem2); + actualProperties.add(properties); + + properties = getItem(UUID.randomUUID().toString()); + CosmosItemResponse upsertResponse2 = encryptionAsyncContainerOriginal.upsertItem(properties, new CosmosItemRequestOptions()).block(); + assertThat(upsertResponse2.getRequestCharge()).isGreaterThan(0); + EncryptionPojo responseItem3 = upsertResponse2.getItem(); + validateResponse(properties, responseItem3); + actualProperties.add(properties); + + //Read Item + EncryptionPojo readItem = encryptionAsyncContainerOriginal.readItem(actualProperties.get(0).getId(), + new PartitionKey(actualProperties.get(0).getMypk()), EncryptionPojo.class).block().getItem(); + validateResponse(actualProperties.get(0), readItem); + + //Query Item + String query = String.format("SELECT * from c where c.id = '%s'", actualProperties.get(1).getId()); + + CosmosPagedFlux feedResponseIterator = + encryptionAsyncContainerOriginal.queryItems(query, EncryptionPojo.class); + List feedResponse = feedResponseIterator.byPage().blockFirst().getResults(); + assertThat(feedResponse.size()).isGreaterThanOrEqualTo(1); + for (EncryptionPojo pojo : feedResponse) { + if (pojo.getId().equals(actualProperties.get(1).getId())) { + validateResponse(pojo, responseItem1); + } } - } - CosmosQueryRequestOptions cosmosQueryRequestOptions1 = new CosmosQueryRequestOptions(); + CosmosQueryRequestOptions cosmosQueryRequestOptions1 = new CosmosQueryRequestOptions(); - CosmosPagedFlux feedResponseIterator1 = - encryptionAsyncContainerOriginal.queryItems(query, cosmosQueryRequestOptions1, EncryptionPojo.class); - List feedResponse1 = feedResponseIterator1.byPage().blockFirst().getResults(); - assertThat(feedResponse1.size()).isGreaterThanOrEqualTo(1); - for (EncryptionPojo pojo : feedResponse1) { - if (pojo.getId().equals(actualProperties.get(1).getId())) { - validateResponse(pojo, responseItem1); + CosmosPagedFlux feedResponseIterator1 = + encryptionAsyncContainerOriginal.queryItems(query, cosmosQueryRequestOptions1, EncryptionPojo.class); + List feedResponse1 = feedResponseIterator1.byPage().blockFirst().getResults(); + assertThat(feedResponse1.size()).isGreaterThanOrEqualTo(1); + for (EncryptionPojo pojo : feedResponse1) { + if (pojo.getId().equals(actualProperties.get(1).getId())) { + validateResponse(pojo, responseItem1); + } } - } - CosmosQueryRequestOptions cosmosQueryRequestOptions2 = new CosmosQueryRequestOptions(); - SqlQuerySpec querySpec = new SqlQuerySpec(query); - - CosmosPagedFlux feedResponseIterator2 = - encryptionAsyncContainerOriginal.queryItems(querySpec, cosmosQueryRequestOptions2, EncryptionPojo.class); - List feedResponse2 = feedResponseIterator2.byPage().blockFirst().getResults(); - assertThat(feedResponse2.size()).isGreaterThanOrEqualTo(1); - for (EncryptionPojo pojo : feedResponse2) { - if (pojo.getId().equals(actualProperties.get(1).getId())) { - validateResponse(pojo, responseItem1); + CosmosQueryRequestOptions cosmosQueryRequestOptions2 = new CosmosQueryRequestOptions(); + SqlQuerySpec querySpec = new SqlQuerySpec(query); + + CosmosPagedFlux feedResponseIterator2 = + encryptionAsyncContainerOriginal.queryItems(querySpec, cosmosQueryRequestOptions2, EncryptionPojo.class); + List feedResponse2 = feedResponseIterator2.byPage().blockFirst().getResults(); + assertThat(feedResponse2.size()).isGreaterThanOrEqualTo(1); + for (EncryptionPojo pojo : feedResponse2) { + if (pojo.getId().equals(actualProperties.get(1).getId())) { + validateResponse(pojo, responseItem1); + } } - } - //Replace Item - CosmosItemResponse replaceResponse = - encryptionAsyncContainerOriginal.replaceItem(actualProperties.get(2), actualProperties.get(2).getId(), - new PartitionKey(actualProperties.get(2).getMypk())).block(); - assertThat(upsertResponse1.getRequestCharge()).isGreaterThan(0); - responseItem = replaceResponse.getItem(); - validateResponse(actualProperties.get(2), responseItem); - - //Delete Item - CosmosItemResponse deleteResponse = encryptionAsyncContainerOriginal.deleteItem(actualProperties.get(0).getId(), - new PartitionKey(actualProperties.get(0).getMypk())).block(); - assertThat(deleteResponse.getStatusCode()).isEqualTo(204); - - CosmosItemResponse deleteResponse1 = encryptionAsyncContainerOriginal.deleteItem(actualProperties.get(1).getId(), - new PartitionKey(actualProperties.get(1).getMypk()), new CosmosItemRequestOptions()).block(); - assertThat(deleteResponse1.getStatusCode()).isEqualTo(204); - - CosmosItemResponse deleteResponse2 = encryptionAsyncContainerOriginal.deleteItem(actualProperties.get(2), - new CosmosItemRequestOptions()).block(); - assertThat(deleteResponse2.getStatusCode()).isEqualTo(204); - - CosmosItemResponse deleteResponse3 = encryptionAsyncContainerOriginal.deleteAllItemsByPartitionKey(new PartitionKey(actualProperties.get(3).getMypk()), - new CosmosItemRequestOptions()).block(); - assertThat(deleteResponse3.getStatusCode()).isEqualTo(200); + //Replace Item + CosmosItemResponse replaceResponse = + encryptionAsyncContainerOriginal.replaceItem(actualProperties.get(2), actualProperties.get(2).getId(), + new PartitionKey(actualProperties.get(2).getMypk())).block(); + assertThat(upsertResponse1.getRequestCharge()).isGreaterThan(0); + responseItem = replaceResponse.getItem(); + validateResponse(actualProperties.get(2), responseItem); + + //Delete Item + CosmosItemResponse deleteResponse = encryptionAsyncContainerOriginal.deleteItem(actualProperties.get(0).getId(), + new PartitionKey(actualProperties.get(0).getMypk())).block(); + assertThat(deleteResponse.getStatusCode()).isEqualTo(204); + + CosmosItemResponse deleteResponse1 = encryptionAsyncContainerOriginal.deleteItem(actualProperties.get(1).getId(), + new PartitionKey(actualProperties.get(1).getMypk()), new CosmosItemRequestOptions()).block(); + assertThat(deleteResponse1.getStatusCode()).isEqualTo(204); + + CosmosItemResponse deleteResponse2 = encryptionAsyncContainerOriginal.deleteItem(actualProperties.get(2), + new CosmosItemRequestOptions()).block(); + assertThat(deleteResponse2.getStatusCode()).isEqualTo(204); + + CosmosItemResponse deleteResponse3 = encryptionAsyncContainerOriginal.deleteAllItemsByPartitionKey(new PartitionKey(actualProperties.get(3).getMypk()), + new CosmosItemRequestOptions()).block(); + assertThat(deleteResponse3.getStatusCode()).isEqualTo(200); + } } finally { try { //deleting the database created for this test From ad522f15ce3d4e5f8d1fb3c565211d3a1aa81ec4 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Thu, 13 Nov 2025 21:24:16 +0100 Subject: [PATCH 48/65] Fix ByteBuf memory leak in TcpServerMock request decoders (#47269) * Initial plan * Fix memory leak in ServerRntbdRequestDecoder and ServerRntbdContextRequestDecoder When overriding channelRead() in ByteToMessageDecoder and bypassing the parent's decode logic by calling context.fireChannelRead() directly, the ByteBuf reference must be explicitly released to prevent memory leaks. Added ReferenceCountUtil.release() in try-finally blocks to properly manage buffer lifecycle. Co-authored-by: FabianMeiswinkel <19165014+FabianMeiswinkel@users.noreply.github.com> * Replace ReferenceCountUtil.release with safeRelease * Replace ReferenceCountUtil.release with safeRelease --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: FabianMeiswinkel <19165014+FabianMeiswinkel@users.noreply.github.com> Co-authored-by: Fabian Meiswinkel --- .../rntbd/ServerRntbdContextRequestDecoder.java | 10 +++++++++- .../TcpServerMock/rntbd/ServerRntbdRequestDecoder.java | 9 ++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/TcpServerMock/rntbd/ServerRntbdContextRequestDecoder.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/TcpServerMock/rntbd/ServerRntbdContextRequestDecoder.java index 4dd92d213297..ff8dae6c33fb 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/TcpServerMock/rntbd/ServerRntbdContextRequestDecoder.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/TcpServerMock/rntbd/ServerRntbdContextRequestDecoder.java @@ -7,6 +7,7 @@ import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.ByteToMessageDecoder; +import io.netty.util.ReferenceCountUtil; import java.util.List; @@ -39,7 +40,14 @@ public void channelRead(final ChannelHandlerContext context, final Object messag return; } } - context.fireChannelRead(message); + + // When bypassing the parent's decode logic, we must release the ByteBuf + // as context.fireChannelRead() doesn't automatically manage reference counts + try { + context.fireChannelRead(message); + } finally { + ReferenceCountUtil.safeRelease(message); + } } /** diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/TcpServerMock/rntbd/ServerRntbdRequestDecoder.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/TcpServerMock/rntbd/ServerRntbdRequestDecoder.java index d9fdaec12fdf..e67faae9edda 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/TcpServerMock/rntbd/ServerRntbdRequestDecoder.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/TcpServerMock/rntbd/ServerRntbdRequestDecoder.java @@ -6,6 +6,7 @@ import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.ByteToMessageDecoder; +import io.netty.util.ReferenceCountUtil; import java.util.List; @@ -34,7 +35,13 @@ public void channelRead(final ChannelHandlerContext context, final Object messag } } - context.fireChannelRead(message); + // When bypassing the parent's decode logic, we must release the ByteBuf + // as context.fireChannelRead() doesn't automatically manage reference counts + try { + context.fireChannelRead(message); + } finally { + ReferenceCountUtil.safeRelease(message); + } } /** From 738d186f22419456042da2d945773369e3977719 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Thu, 13 Nov 2025 20:33:33 +0000 Subject: [PATCH 49/65] Update ThinClientStoreModel.java --- .../azure/cosmos/implementation/ThinClientStoreModel.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ThinClientStoreModel.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ThinClientStoreModel.java index 2a92039ddd7e..6e3add8f805a 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ThinClientStoreModel.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ThinClientStoreModel.java @@ -135,17 +135,13 @@ public StoreResponse unwrapToStoreResponse( payloadBuf ); - if (payloadBuf != content) { - // payload is a slice/derived view; super() owns payload, we still own the container - // this includes scenarios where payloadBuf == EMPTY_BUFFER + if (payloadBuf == Unpooled.EMPTY_BUFFER) { ReferenceCountUtil.safeRelease(content); } return storeResponse; } catch (Throwable t){ - if (payloadBuf != content) { - // payload is a slice/derived view; super() owns payload, we still own the container - // this includes scenarios where payloadBuf == EMPTY_BUFFER + if (payloadBuf == Unpooled.EMPTY_BUFFER) { ReferenceCountUtil.safeRelease(content); } From db3fb7910c2e9363d6ec15c8b3cd97402121609a Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Thu, 13 Nov 2025 21:21:36 +0000 Subject: [PATCH 50/65] Fixing test issues --- .../cpu/CpuLoadMonitorTest.java | 24 ++++++++++++------- .../http/ReactorNettyHttpClientTest.java | 8 ++++++- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/cpu/CpuLoadMonitorTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/cpu/CpuLoadMonitorTest.java index 46b5e0374438..ceafb8c65778 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/cpu/CpuLoadMonitorTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/cpu/CpuLoadMonitorTest.java @@ -15,7 +15,7 @@ public class CpuLoadMonitorTest { @Test(groups = "unit") public void noInstance() throws Exception { - assertThat(ReflectionUtils.getListeners()).hasSize(0); + assertEventualListenerCount(0); assertThat(ReflectionUtils.getFuture()).isNull(); } @@ -31,7 +31,7 @@ public void multipleInstances() throws Exception { Future workFuture = ReflectionUtils.getFuture(); assertThat(workFuture).isNotNull(); assertThat(workFuture.isCancelled()).isFalse(); - assertThat(ReflectionUtils.getListeners()).hasSize(cpuMonitorList.size()); + assertEventualListenerCount(cpuMonitorList.size()); Thread.sleep(10); } @@ -41,7 +41,7 @@ public void multipleInstances() throws Exception { CpuMemoryMonitor.unregister(cpuMemoryListener); - assertThat(ReflectionUtils.getListeners()).hasSize(cpuMonitorList.size()); + assertEventualListenerCount(cpuMonitorList.size()); Future workFuture = ReflectionUtils.getFuture(); assertThat(workFuture).isNotNull(); @@ -54,7 +54,7 @@ public void multipleInstances() throws Exception { CpuMemoryMonitor.register(newListener); CpuMemoryMonitor.unregister(newListener); - assertThat(ReflectionUtils.getListeners()).hasSize(cpuMonitorList.size()); + assertEventualListenerCount(cpuMonitorList.size()); Future workFuture = ReflectionUtils.getFuture(); assertThat(workFuture).isNotNull(); assertThat(workFuture.isCancelled()).isFalse(); @@ -62,7 +62,7 @@ public void multipleInstances() throws Exception { CpuMemoryListener cpuMemoryListener = cpuMonitorList.remove(0); CpuMemoryMonitor.unregister(cpuMemoryListener); - assertThat(ReflectionUtils.getListeners()).hasSize(cpuMonitorList.size()); + assertEventualListenerCount(cpuMonitorList.size()); workFuture = ReflectionUtils.getFuture(); assertThat(workFuture).isNull(); @@ -74,12 +74,20 @@ public void handleLeak() throws Throwable { CpuMemoryMonitor.register(listener); listener = null; System.gc(); - Thread.sleep(10000); - - assertThat(ReflectionUtils.getListeners()).hasSize(0); + int secondsWaited = 0; + assertEventualListenerCount(0); assertThat(ReflectionUtils.getFuture()).isNull(); } + private static void assertEventualListenerCount(int expectedListenerCount) throws Exception { + int secondsWaited = 0; + while (secondsWaited < 30 && ReflectionUtils.getListeners().size() > expectedListenerCount) { + Thread.sleep(1_000); + } + + assertThat(ReflectionUtils.getListeners()).hasSize(expectedListenerCount); + } + class TestMemoryListener implements CpuMemoryListener { } } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/http/ReactorNettyHttpClientTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/http/ReactorNettyHttpClientTest.java index 38991fa0f044..88b85abbdf68 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/http/ReactorNettyHttpClientTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/http/ReactorNettyHttpClientTest.java @@ -29,15 +29,21 @@ public class ReactorNettyHttpClientTest { private static final Logger logger = LoggerFactory.getLogger(ReactorNettyHttpClientTest.class); private HttpClient reactorNettyHttpClient; + private volatile AutoCloseable disableNettyLeakDetectionScope; @BeforeClass(groups = "unit") public void before_ReactorNettyHttpClientTest() { this.reactorNettyHttpClient = HttpClient.createFixed(new HttpClientConfig(new Configs())); + this.disableNettyLeakDetectionScope = CosmosNettyLeakDetectorFactory.createDisableLeakDetectionScope(); } @AfterClass(groups = "unit", alwaysRun = true) - public void after_ReactorNettyHttpClientTest() { + public void after_ReactorNettyHttpClientTest() throws Exception { + LifeCycleUtils.closeQuietly(reactorNettyHttpClient); + if (this.disableNettyLeakDetectionScope != null) { + this.disableNettyLeakDetectionScope.close(); + } } @Test(groups = "unit") From 3b2a74ff79227a688de8c351f618267fbea20b44 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Thu, 13 Nov 2025 21:33:22 +0000 Subject: [PATCH 51/65] Disable netty leak detection in RetrycontextOnDiagnosticTest --- .../cosmos/RetryContextOnDiagnosticTest.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/RetryContextOnDiagnosticTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/RetryContextOnDiagnosticTest.java index f0d46402a0b4..b6190b00f116 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/RetryContextOnDiagnosticTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/RetryContextOnDiagnosticTest.java @@ -64,6 +64,8 @@ import io.reactivex.subscribers.TestSubscriber; import org.mockito.ArgumentMatchers; import org.mockito.Mockito; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; import reactor.core.publisher.Mono; @@ -94,6 +96,19 @@ public class RetryContextOnDiagnosticTest extends TestSuiteBase { private IRetryPolicy retryPolicy; private RxDocumentServiceRequest serviceRequest; private AddressSelector addressSelector; + private volatile AutoCloseable disableNettyLeakDetectionScope; + + @BeforeClass(groups = "unit") + public void beforeClass_DisableNettyLeakDetection() { + this.disableNettyLeakDetectionScope = CosmosNettyLeakDetectorFactory.createDisableLeakDetectionScope(); + } + + @AfterClass(groups = "unit", alwaysRun = true) + public void afterClass_ReactivateNettyLeakDetection() throws Exception { + if (this.disableNettyLeakDetectionScope != null) { + this.disableNettyLeakDetectionScope.close(); + } + } @Test(groups = {"unit"}, timeOut = TIMEOUT * 2) public void backoffRetryUtilityExecuteRetry() throws Exception { From 3e53ed7c440a07fa3c7b2f435061d302d08bc36d Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Fri, 14 Nov 2025 00:21:47 +0000 Subject: [PATCH 52/65] Test and diagnostics improvements --- .../implementation/ConsistencyTests1.java | 376 ++++++++++-------- .../implementation/ConsistencyTests2.java | 136 ++++--- .../implementation/ConsistencyTestsBase.java | 122 +++--- .../ConnectionStateListenerTest.java | 27 +- .../ServerRntbdContextRequestDecoder.java | 8 +- .../rntbd/ServerRntbdRequestDecoder.java | 8 +- .../rntbd/RntbdContextDecoder.java | 28 ++ .../rntbd/RntbdContextRequestDecoder.java | 34 ++ .../rntbd/RntbdRequestDecoder.java | 34 ++ .../rntbd/RntbdResponseDecoder.java | 34 ++ 10 files changed, 509 insertions(+), 298 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConsistencyTests1.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConsistencyTests1.java index 22cba6842472..de8120a84091 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConsistencyTests1.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConsistencyTests1.java @@ -28,34 +28,42 @@ public void validateStrongConsistencyOnSyncReplication() throws Exception { throw new SkipException("Endpoint does not have strong consistency"); } - ConnectionPolicy connectionPolicy = new ConnectionPolicy(GatewayConnectionConfig.getDefaultConfig()); - this.writeClient = - (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withConsistencyLevel(ConsistencyLevel.STRONG) - .withContentResponseOnWriteEnabled(true) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); + RxDocumentClientImpl writeClient = null; + RxDocumentClientImpl readClient = null; - this.readClient = + try { + ConnectionPolicy connectionPolicy = new ConnectionPolicy(GatewayConnectionConfig.getDefaultConfig()); + writeClient = (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withConsistencyLevel(ConsistencyLevel.STRONG) - .withContentResponseOnWriteEnabled(true) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); - User userDefinition = getUserDefinition(); - userDefinition.setId(userDefinition.getId() + "validateStrongConsistencyOnSyncReplication"); - User user = safeCreateUser(this.initClient, createdDatabase.getId(), userDefinition); - validateStrongConsistency(user); + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withConsistencyLevel(ConsistencyLevel.STRONG) + .withContentResponseOnWriteEnabled(true) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); + + readClient = + (RxDocumentClientImpl) new AsyncDocumentClient.Builder() + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withConsistencyLevel(ConsistencyLevel.STRONG) + .withContentResponseOnWriteEnabled(true) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); + User userDefinition = getUserDefinition(); + userDefinition.setId(userDefinition.getId() + "validateStrongConsistencyOnSyncReplication"); + User user = safeCreateUser(this.initClient, createdDatabase.getId(), userDefinition); + validateStrongConsistency(user, readClient, writeClient); + } finally { + safeClose(readClient); + safeClose(writeClient); + } } @@ -63,85 +71,109 @@ public void validateStrongConsistencyOnSyncReplication() throws Exception { public void validateConsistentLSNForDirectTCPClient() { //TODO Need to test with TCP protocol // https://msdata.visualstudio.com/CosmosDB/_workitems/edit/355057 - ConnectionPolicy connectionPolicy = new ConnectionPolicy(DirectConnectionConfig.getDefaultConfig()); - this.writeClient = - (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withContentResponseOnWriteEnabled(true) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); + RxDocumentClientImpl writeClient = null; + RxDocumentClientImpl readClient = null; - this.readClient = + try { + ConnectionPolicy connectionPolicy = new ConnectionPolicy(DirectConnectionConfig.getDefaultConfig()); + writeClient = (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withContentResponseOnWriteEnabled(true) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); - validateConsistentLSN(); + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withContentResponseOnWriteEnabled(true) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); + + readClient = + (RxDocumentClientImpl) new AsyncDocumentClient.Builder() + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withContentResponseOnWriteEnabled(true) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); + validateConsistentLSN(readClient, writeClient); + } finally { + safeClose(readClient); + safeClose(writeClient); + } } @Test(groups = {"direct"}, timeOut = CONSISTENCY_TEST_TIMEOUT) public void validateConsistentLSNForDirectHttpsClient() { ConnectionPolicy connectionPolicy = new ConnectionPolicy(DirectConnectionConfig.getDefaultConfig()); - this.writeClient = - (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withContentResponseOnWriteEnabled(true) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); + RxDocumentClientImpl writeClient = null; + RxDocumentClientImpl readClient = null; - this.readClient = + try { + writeClient = (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withContentResponseOnWriteEnabled(true) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); - validateConsistentLSN(); + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withContentResponseOnWriteEnabled(true) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); + + readClient = + (RxDocumentClientImpl) new AsyncDocumentClient.Builder() + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withContentResponseOnWriteEnabled(true) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); + validateConsistentLSN(readClient, writeClient); + } finally { + safeClose(readClient); + safeClose(writeClient); + } } @Test(groups = {"direct"}, timeOut = CONSISTENCY_TEST_TIMEOUT, enabled = false) public void validateConsistentLSNAndQuorumAckedLSNForDirectTCPClient() { //TODO Need to test with TCP protocol //https://msdata.visualstudio.com/CosmosDB/_workitems/edit/355057 - ConnectionPolicy connectionPolicy = new ConnectionPolicy(DirectConnectionConfig.getDefaultConfig()); - this.writeClient = - (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withContentResponseOnWriteEnabled(true) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); + RxDocumentClientImpl writeClient = null; + RxDocumentClientImpl readClient = null; - this.readClient = + try { + ConnectionPolicy connectionPolicy = new ConnectionPolicy(DirectConnectionConfig.getDefaultConfig()); + writeClient = (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withContentResponseOnWriteEnabled(true) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); - validateConsistentLSNAndQuorumAckedLSN(); + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withContentResponseOnWriteEnabled(true) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); + + readClient = + (RxDocumentClientImpl) new AsyncDocumentClient.Builder() + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withContentResponseOnWriteEnabled(true) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); + validateConsistentLSNAndQuorumAckedLSN(readClient, writeClient); + } finally { + safeClose(readClient); + safeClose(writeClient); + } } @Test(groups = {"direct"}, timeOut = CONSISTENCY_TEST_TIMEOUT) @@ -164,29 +196,37 @@ public void validateBoundedStalenessDynamicQuorumSyncReplication() { @Test(groups = {"direct"}, timeOut = CONSISTENCY_TEST_TIMEOUT) public void validateConsistentLSNAndQuorumAckedLSNForDirectHttpsClient() { - ConnectionPolicy connectionPolicy = new ConnectionPolicy(DirectConnectionConfig.getDefaultConfig()); - this.writeClient = - (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withContentResponseOnWriteEnabled(true) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); + RxDocumentClientImpl writeClient = null; + RxDocumentClientImpl readClient = null; - this.readClient = + try { + ConnectionPolicy connectionPolicy = new ConnectionPolicy(DirectConnectionConfig.getDefaultConfig()); + writeClient = (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withContentResponseOnWriteEnabled(true) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); - validateConsistentLSNAndQuorumAckedLSN(); + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withContentResponseOnWriteEnabled(true) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); + + readClient = + (RxDocumentClientImpl) new AsyncDocumentClient.Builder() + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withContentResponseOnWriteEnabled(true) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); + validateConsistentLSNAndQuorumAckedLSN(readClient, writeClient); + } finally { + safeClose(readClient); + safeClose(writeClient); + } } // TODO (DANOBLE) test is flaky @@ -218,33 +258,41 @@ public void validateConsistentPrefixOnSyncReplication() throws InterruptedExcept throw new SkipException("Endpoint does not have strong consistency"); } - ConnectionPolicy connectionPolicy = new ConnectionPolicy(GatewayConnectionConfig.getDefaultConfig()); - this.writeClient = - (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withConsistencyLevel(ConsistencyLevel.BOUNDED_STALENESS) - .withContentResponseOnWriteEnabled(true) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); + RxDocumentClientImpl writeClient = null; + RxDocumentClientImpl readClient = null; - this.readClient = + try { + ConnectionPolicy connectionPolicy = new ConnectionPolicy(GatewayConnectionConfig.getDefaultConfig()); + writeClient = (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withConsistencyLevel(ConsistencyLevel.BOUNDED_STALENESS) - .withContentResponseOnWriteEnabled(true) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); - User user = safeCreateUser(this.initClient, createdDatabase.getId(), getUserDefinition()); - boolean readLagging = validateConsistentPrefix(user); - assertThat(readLagging).isFalse(); + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withConsistencyLevel(ConsistencyLevel.BOUNDED_STALENESS) + .withContentResponseOnWriteEnabled(true) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); + + readClient = + (RxDocumentClientImpl) new AsyncDocumentClient.Builder() + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withConsistencyLevel(ConsistencyLevel.BOUNDED_STALENESS) + .withContentResponseOnWriteEnabled(true) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); + User user = safeCreateUser(this.initClient, createdDatabase.getId(), getUserDefinition()); + boolean readLagging = validateConsistentPrefix(user, readClient, writeClient); + assertThat(readLagging).isFalse(); + } finally { + safeClose(readClient); + safeClose(writeClient); + } } @Test(groups = {"direct"}, timeOut = CONSISTENCY_TEST_TIMEOUT) @@ -253,34 +301,42 @@ public void validateConsistentPrefixOnAsyncReplication() throws InterruptedExcep throw new SkipException("Endpoint does not have strong consistency"); } - ConnectionPolicy connectionPolicy = new ConnectionPolicy(DirectConnectionConfig.getDefaultConfig()); - this.writeClient = - (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withConsistencyLevel(ConsistencyLevel.BOUNDED_STALENESS) - .withContentResponseOnWriteEnabled(true) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); + RxDocumentClientImpl writeClient = null; + RxDocumentClientImpl readClient = null; - this.readClient = + try { + ConnectionPolicy connectionPolicy = new ConnectionPolicy(DirectConnectionConfig.getDefaultConfig()); + writeClient = (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withConsistencyLevel(ConsistencyLevel.BOUNDED_STALENESS) - .withContentResponseOnWriteEnabled(true) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); - Document documentDefinition = getDocumentDefinition(); - Document document = createDocument(this.initClient, createdDatabase.getId(), createdCollection.getId(), documentDefinition); - boolean readLagging = validateConsistentPrefix(document); - //assertThat(readLagging).isTrue(); //Will fail if batch repl is turned off + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withConsistencyLevel(ConsistencyLevel.BOUNDED_STALENESS) + .withContentResponseOnWriteEnabled(true) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); + + readClient = + (RxDocumentClientImpl) new AsyncDocumentClient.Builder() + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withConsistencyLevel(ConsistencyLevel.BOUNDED_STALENESS) + .withContentResponseOnWriteEnabled(true) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); + Document documentDefinition = getDocumentDefinition(); + Document document = createDocument(this.initClient, createdDatabase.getId(), createdCollection.getId(), documentDefinition); + boolean readLagging = validateConsistentPrefix(document, readClient, writeClient); + //assertThat(readLagging).isTrue(); //Will fail if batch repl is turned off + } finally { + safeClose(readClient); + safeClose(writeClient); + } } @Test(groups = {"direct"}, timeOut = CONSISTENCY_TEST_TIMEOUT, enabled = false) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConsistencyTests2.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConsistencyTests2.java index 1143dfcba0e8..4bc97ff928dc 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConsistencyTests2.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConsistencyTests2.java @@ -42,75 +42,91 @@ public Object[] regionScopedSessionContainerConfigs() { @Test(groups = {"direct"}, dataProvider = "regionScopedSessionContainerConfigs", timeOut = CONSISTENCY_TEST_TIMEOUT) public void validateReadSessionOnAsyncReplication(boolean shouldRegionScopedSessionContainerEnabled) throws InterruptedException { - ConnectionPolicy connectionPolicy = new ConnectionPolicy(GatewayConnectionConfig.getDefaultConfig()); - this.writeClient = - (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withConsistencyLevel(ConsistencyLevel.SESSION) - .withContentResponseOnWriteEnabled(true) - .withRegionScopedSessionCapturingEnabled(shouldRegionScopedSessionContainerEnabled) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); + RxDocumentClientImpl writeClient = null; + RxDocumentClientImpl readClient = null; - this.readClient = + try { + ConnectionPolicy connectionPolicy = new ConnectionPolicy(GatewayConnectionConfig.getDefaultConfig()); + writeClient = (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withConsistencyLevel(ConsistencyLevel.SESSION) - .withContentResponseOnWriteEnabled(true) - .withRegionScopedSessionCapturingEnabled(shouldRegionScopedSessionContainerEnabled) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); - - Document document = this.initClient.createDocument(createdCollection.getSelfLink(), getDocumentDefinition(), - null, false).block().getResource(); - Thread.sleep(5000);//WaitForServerReplication - boolean readLagging = this.validateReadSession(document); - //assertThat(readLagging).isTrue(); //Will fail if batch repl is turned off + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withConsistencyLevel(ConsistencyLevel.SESSION) + .withContentResponseOnWriteEnabled(true) + .withRegionScopedSessionCapturingEnabled(shouldRegionScopedSessionContainerEnabled) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); + + readClient = + (RxDocumentClientImpl) new AsyncDocumentClient.Builder() + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withConsistencyLevel(ConsistencyLevel.SESSION) + .withContentResponseOnWriteEnabled(true) + .withRegionScopedSessionCapturingEnabled(shouldRegionScopedSessionContainerEnabled) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); + + Document document = this.initClient.createDocument(createdCollection.getSelfLink(), getDocumentDefinition(), + null, false).block().getResource(); + Thread.sleep(5000);//WaitForServerReplication + boolean readLagging = this.validateReadSession(document, readClient, writeClient); + //assertThat(readLagging).isTrue(); //Will fail if batch repl is turned off + } finally { + safeClose(readClient); + safeClose(writeClient); + } } @Test(groups = {"direct"}, dataProvider = "regionScopedSessionContainerConfigs", timeOut = CONSISTENCY_TEST_TIMEOUT) public void validateWriteSessionOnAsyncReplication(boolean shouldRegionScopedSessionContainerEnabled) throws InterruptedException { - ConnectionPolicy connectionPolicy = new ConnectionPolicy(GatewayConnectionConfig.getDefaultConfig()); - this.writeClient = - (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withConsistencyLevel(ConsistencyLevel.SESSION) - .withContentResponseOnWriteEnabled(true) - .withRegionScopedSessionCapturingEnabled(shouldRegionScopedSessionContainerEnabled) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); + RxDocumentClientImpl writeClient = null; + RxDocumentClientImpl readClient = null; - this.readClient = + try { + ConnectionPolicy connectionPolicy = new ConnectionPolicy(GatewayConnectionConfig.getDefaultConfig()); + writeClient = (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withConsistencyLevel(ConsistencyLevel.SESSION) - .withContentResponseOnWriteEnabled(true) - .withRegionScopedSessionCapturingEnabled(shouldRegionScopedSessionContainerEnabled) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); - - Document document = this.initClient.createDocument(createdCollection.getSelfLink(), getDocumentDefinition(), - null, false).block().getResource(); - Thread.sleep(5000);//WaitForServerReplication - boolean readLagging = this.validateWriteSession(document); - //assertThat(readLagging).isTrue(); //Will fail if batch repl is turned off + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withConsistencyLevel(ConsistencyLevel.SESSION) + .withContentResponseOnWriteEnabled(true) + .withRegionScopedSessionCapturingEnabled(shouldRegionScopedSessionContainerEnabled) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); + + readClient = + (RxDocumentClientImpl) new AsyncDocumentClient.Builder() + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withConsistencyLevel(ConsistencyLevel.SESSION) + .withContentResponseOnWriteEnabled(true) + .withRegionScopedSessionCapturingEnabled(shouldRegionScopedSessionContainerEnabled) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); + + Document document = this.initClient.createDocument(createdCollection.getSelfLink(), getDocumentDefinition(), + null, false).block().getResource(); + Thread.sleep(5000);//WaitForServerReplication + boolean readLagging = this.validateWriteSession(document, readClient, writeClient); + //assertThat(readLagging).isTrue(); //Will fail if batch repl is turned off + } finally { + safeClose(readClient); + safeClose(writeClient); + } } @Test(groups = {"direct"}, timeOut = CONSISTENCY_TEST_TIMEOUT, enabled = false) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConsistencyTestsBase.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConsistencyTestsBase.java index 3b5244997d2e..75b5c04bf6ea 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConsistencyTestsBase.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/ConsistencyTestsBase.java @@ -36,8 +36,6 @@ public class ConsistencyTestsBase extends TestSuiteBase { static final int CONSISTENCY_TEST_TIMEOUT = 120000; static final String USER_NAME = "TestUser"; - RxDocumentClientImpl writeClient; - RxDocumentClientImpl readClient; AsyncDocumentClient initClient; Database createdDatabase; DocumentCollection createdCollection; @@ -49,7 +47,11 @@ public void before_ConsistencyTestsBase() throws Exception { createdCollection = SHARED_MULTI_PARTITION_COLLECTION; } - void validateStrongConsistency(Resource resourceToWorkWith) throws Exception { + void validateStrongConsistency( + Resource resourceToWorkWith, + RxDocumentClientImpl readClient, + RxDocumentClientImpl writeClient) throws Exception { + int numberOfTestIteration = 5; Resource writeResource = resourceToWorkWith; while (numberOfTestIteration-- > 0) //Write from a client and do point read through second client and ensure TS matches. @@ -58,47 +60,47 @@ void validateStrongConsistency(Resource resourceToWorkWith) throws Exception { Thread.sleep(1000); //Timestamp is in granularity of seconds. Resource updatedResource = null; if (resourceToWorkWith instanceof User) { - updatedResource = this.writeClient.upsertUser(createdDatabase.getSelfLink(), (User) writeResource, null).block().getResource(); + updatedResource = writeClient.upsertUser(createdDatabase.getSelfLink(), (User) writeResource, null).block().getResource(); } else if (resourceToWorkWith instanceof Document) { RequestOptions options = new RequestOptions(); options.setPartitionKey(new PartitionKey(resourceToWorkWith.get("mypk"))); - updatedResource = this.writeClient.upsertDocument(createdCollection.getSelfLink(), (Document) writeResource, options, false).block().getResource(); + updatedResource = writeClient.upsertDocument(createdCollection.getSelfLink(), (Document) writeResource, options, false).block().getResource(); } assertThat(updatedResource.getTimestamp().isAfter(sourceTimestamp)).isTrue(); - User readResource = this.readClient.readUser(resourceToWorkWith.getSelfLink(), null).block().getResource(); + User readResource = readClient.readUser(resourceToWorkWith.getSelfLink(), null).block().getResource(); assertThat(updatedResource.getTimestamp().equals(readResource.getTimestamp())); } } - void validateConsistentLSN() { + void validateConsistentLSN(RxDocumentClientImpl readClient, RxDocumentClientImpl writeClient) { Document documentDefinition = getDocumentDefinition(); RequestOptions options = new RequestOptions(); options.setPartitionKey(new PartitionKey(documentDefinition.get("mypk"))); - Document document = createDocument(this.writeClient, createdDatabase.getId(), createdCollection.getId(), documentDefinition); - ResourceResponse response = this.writeClient.deleteDocument(document.getSelfLink(), options).block(); + Document document = createDocument(writeClient, createdDatabase.getId(), createdCollection.getId(), documentDefinition); + ResourceResponse response = writeClient.deleteDocument(document.getSelfLink(), options).block(); assertThat(response.getStatusCode()).isEqualTo(204); long quorumAckedLSN = Long.parseLong(response.getResponseHeaders().get(WFConstants.BackendHeaders.QUORUM_ACKED_LSN)); assertThat(quorumAckedLSN > 0).isTrue(); FailureValidator validator = new FailureValidator.Builder().statusCode(404).lsnGreaterThan(quorumAckedLSN).build(); - Mono> readObservable = this.readClient.readDocument(document.getSelfLink(), options); + Mono> readObservable = readClient.readDocument(document.getSelfLink(), options); validateFailure(readObservable, validator); } - void validateConsistentLSNAndQuorumAckedLSN() { + void validateConsistentLSNAndQuorumAckedLSN(RxDocumentClientImpl readClient, RxDocumentClientImpl writeClient) { Document documentDefinition = getDocumentDefinition(); RequestOptions options = new RequestOptions(); options.setPartitionKey(new PartitionKey(documentDefinition.get("mypk"))); - Document document = createDocument(this.writeClient, createdDatabase.getId(), createdCollection.getId(), documentDefinition); - ResourceResponse response = this.writeClient.deleteDocument(document.getSelfLink(), options).block(); + Document document = createDocument(writeClient, createdDatabase.getId(), createdCollection.getId(), documentDefinition); + ResourceResponse response = writeClient.deleteDocument(document.getSelfLink(), options).block(); assertThat(response.getStatusCode()).isEqualTo(204); long quorumAckedLSN = Long.parseLong(response.getResponseHeaders().get(WFConstants.BackendHeaders.QUORUM_ACKED_LSN)); assertThat(quorumAckedLSN > 0).isTrue(); FailureValidator validator = new FailureValidator.Builder().statusCode(404).lsnGreaterThanEqualsTo(quorumAckedLSN).exceptionQuorumAckedLSNInNotNull().build(); - Mono> readObservable = this.readClient.deleteDocument(document.getSelfLink(), options); + Mono> readObservable = readClient.deleteDocument(document.getSelfLink(), options); validateFailure(readObservable, validator); } @@ -119,36 +121,48 @@ void validateStrongConsistencyOnAsyncReplication(boolean useGateway) throws Inte connectionPolicy = new ConnectionPolicy(GatewayConnectionConfig.getDefaultConfig()); } - this.writeClient = + RxDocumentClientImpl writeClient = null; + RxDocumentClientImpl readClient = null; + + try { + writeClient = (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withConsistencyLevel(ConsistencyLevel.STRONG) - .withContentResponseOnWriteEnabled(true) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withConsistencyLevel(ConsistencyLevel.STRONG) + .withContentResponseOnWriteEnabled(true) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); - this.readClient = + readClient = (RxDocumentClientImpl) new AsyncDocumentClient.Builder() - .withServiceEndpoint(TestConfigurations.HOST) - .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) - .withConnectionPolicy(connectionPolicy) - .withConsistencyLevel(ConsistencyLevel.STRONG) - .withContentResponseOnWriteEnabled(true) - .withClientTelemetryConfig( - new CosmosClientTelemetryConfig() - .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) - .build(); + .withServiceEndpoint(TestConfigurations.HOST) + .withMasterKeyOrResourceToken(TestConfigurations.MASTER_KEY) + .withConnectionPolicy(connectionPolicy) + .withConsistencyLevel(ConsistencyLevel.STRONG) + .withContentResponseOnWriteEnabled(true) + .withClientTelemetryConfig( + new CosmosClientTelemetryConfig() + .sendClientTelemetryToService(ClientTelemetry.DEFAULT_CLIENT_TELEMETRY_ENABLED)) + .build(); - Document documentDefinition = getDocumentDefinition(); - Document document = createDocument(this.writeClient, createdDatabase.getId(), createdCollection.getId(), documentDefinition); - validateStrongConsistency(document, TestUtils.getCollectionNameLink(createdDatabase.getId(), createdCollection.getId())); + Document documentDefinition = getDocumentDefinition(); + Document document = createDocument(writeClient, createdDatabase.getId(), createdCollection.getId(), documentDefinition); + validateStrongConsistency( + document, + TestUtils.getCollectionNameLink(createdDatabase.getId(), createdCollection.getId()), + readClient, + writeClient); + } finally { + safeClose(readClient); + safeClose(writeClient); + } } - void validateStrongConsistency(Document documentToWorkWith, String collectionLink) throws InterruptedException { + void validateStrongConsistency(Document documentToWorkWith, String collectionLink, RxDocumentClientImpl readClient, RxDocumentClientImpl writeClient) throws InterruptedException { int numberOfTestIteration = 5; Document writeDocument = documentToWorkWith; while (numberOfTestIteration-- > 0) { @@ -156,10 +170,10 @@ void validateStrongConsistency(Document documentToWorkWith, String collectionLin Thread.sleep(1000);//Timestamp is in granularity of seconds. RequestOptions options = new RequestOptions(); options.setPartitionKey(new PartitionKey(documentToWorkWith.get("mypk"))); - Document updatedDocument = this.writeClient.replaceDocument(writeDocument, options).block().getResource(); + Document updatedDocument = writeClient.replaceDocument(writeDocument, options).block().getResource(); assertThat(updatedDocument.getTimestamp().isAfter(sourceTimestamp)).isTrue(); - Document readDocument = this.readClient.readDocument(documentToWorkWith.getSelfLink(), options).block().getResource(); + Document readDocument = readClient.readDocument(documentToWorkWith.getSelfLink(), options).block().getResource(); assertThat(updatedDocument.getTimestamp().equals(readDocument.getTimestamp())); } } @@ -251,7 +265,7 @@ void validateSessionContainerAfterCollectionCreateReplace(boolean useGateway) { } } - boolean validateConsistentPrefix(Resource resourceToWorkWith) throws InterruptedException { + boolean validateConsistentPrefix(Resource resourceToWorkWith, RxDocumentClientImpl readClient, RxDocumentClientImpl writeClient) throws InterruptedException { int numberOfTestIteration = 5; Instant lastReadDateTime = resourceToWorkWith.getTimestamp(); boolean readLagging = false; @@ -262,12 +276,12 @@ boolean validateConsistentPrefix(Resource resourceToWorkWith) throws Interrupted Thread.sleep(1000); //Timestamp is in granularity of seconds. Resource updatedResource = null; if (resourceToWorkWith instanceof User) { - updatedResource = this.writeClient.upsertUser(createdDatabase.getSelfLink(), (User) writeResource, + updatedResource = writeClient.upsertUser(createdDatabase.getSelfLink(), (User) writeResource, null) .block() .getResource(); } else if (resourceToWorkWith instanceof Document) { - updatedResource = this.writeClient.upsertDocument(createdCollection.getSelfLink(), + updatedResource = writeClient.upsertDocument(createdCollection.getSelfLink(), (Document) writeResource, null, false) .block() .getResource(); @@ -277,13 +291,13 @@ boolean validateConsistentPrefix(Resource resourceToWorkWith) throws Interrupted Resource readResource = null; if (resourceToWorkWith instanceof User) { - readResource = this.readClient.readUser(resourceToWorkWith.getSelfLink(), null) + readResource = readClient.readUser(resourceToWorkWith.getSelfLink(), null) .block() .getResource(); } else if (resourceToWorkWith instanceof Document) { RequestOptions options = new RequestOptions(); options.setPartitionKey(new PartitionKey(resourceToWorkWith.get("mypk"))); - readResource = this.readClient.readDocument(resourceToWorkWith.getSelfLink(), options) + readResource = readClient.readDocument(resourceToWorkWith.getSelfLink(), options) .block() .getResource(); } @@ -296,7 +310,7 @@ boolean validateConsistentPrefix(Resource resourceToWorkWith) throws Interrupted return readLagging; } - boolean validateReadSession(Resource resourceToWorkWith) throws InterruptedException { + boolean validateReadSession(Resource resourceToWorkWith, RxDocumentClientImpl readClient, RxDocumentClientImpl writeClient) throws InterruptedException { int numberOfTestIteration = 5; Instant lastReadDateTime = Instant.MIN; boolean readLagging = false; @@ -307,7 +321,7 @@ boolean validateReadSession(Resource resourceToWorkWith) throws InterruptedExcep Thread.sleep(1000); Resource updatedResource = null; if (resourceToWorkWith instanceof Document) { - updatedResource = this.writeClient.upsertDocument(createdCollection.getSelfLink(), writeResource, + updatedResource = writeClient.upsertDocument(createdCollection.getSelfLink(), writeResource, null, false) .block() .getResource(); @@ -319,7 +333,7 @@ boolean validateReadSession(Resource resourceToWorkWith) throws InterruptedExcep RequestOptions requestOptions = new RequestOptions(); requestOptions.setPartitionKey(new PartitionKey(resourceToWorkWith.get("mypk"))); if (resourceToWorkWith instanceof Document) { - readResource = this.readClient.readDocument(resourceToWorkWith.getSelfLink(), requestOptions).block().getResource(); + readResource = readClient.readDocument(resourceToWorkWith.getSelfLink(), requestOptions).block().getResource(); } assertThat(readResource.getTimestamp().compareTo(lastReadDateTime) >= 0).isTrue(); lastReadDateTime = readResource.getTimestamp(); @@ -331,7 +345,7 @@ boolean validateReadSession(Resource resourceToWorkWith) throws InterruptedExcep return readLagging; } - boolean validateWriteSession(Resource resourceToWorkWith) throws InterruptedException { + boolean validateWriteSession(Resource resourceToWorkWith, RxDocumentClientImpl readClient, RxDocumentClientImpl writeClient) throws InterruptedException { int numberOfTestIteration = 5; Instant lastReadDateTime = Instant.MIN; boolean readLagging = false; @@ -342,7 +356,7 @@ boolean validateWriteSession(Resource resourceToWorkWith) throws InterruptedExce Thread.sleep(1000); Resource updatedResource = null; if (resourceToWorkWith instanceof Document) { - updatedResource = this.writeClient.upsertDocument(createdCollection.getSelfLink(), writeResource, null, false).block().getResource(); + updatedResource = writeClient.upsertDocument(createdCollection.getSelfLink(), writeResource, null, false).block().getResource(); } assertThat(updatedResource.getTimestamp().isAfter(sourceTimestamp)).isTrue(); writeResource = updatedResource; @@ -352,7 +366,7 @@ boolean validateWriteSession(Resource resourceToWorkWith) throws InterruptedExce requestOptions.setPartitionKey(new PartitionKey(resourceToWorkWith.get("mypk"))); if (resourceToWorkWith instanceof Document) { readResource = - this.readClient.readDocument(resourceToWorkWith.getSelfLink(), requestOptions) + readClient.readDocument(resourceToWorkWith.getSelfLink(), requestOptions) .block() .getResource(); } @@ -366,7 +380,7 @@ boolean validateWriteSession(Resource resourceToWorkWith) throws InterruptedExce //Now perform write on session and update our session token and lastReadTS Thread.sleep(1000); if (resourceToWorkWith instanceof Document) { - readResource = this.writeClient.upsertDocument(createdCollection.getSelfLink(), readResource, + readResource = writeClient.upsertDocument(createdCollection.getSelfLink(), readResource, requestOptions, false) .block() .getResource(); @@ -374,7 +388,7 @@ boolean validateWriteSession(Resource resourceToWorkWith) throws InterruptedExce } assertThat(readResource.getTimestamp().isAfter(lastReadDateTime)); - this.readClient.setSession(this.writeClient.getSession()); + readClient.setSession(writeClient.getSession()); } return readLagging; } @@ -873,8 +887,6 @@ private static String getGlobalSessionToken(RxDocumentClientImpl client, Documen @AfterClass(groups = {"direct"}, timeOut = SHUTDOWN_TIMEOUT, alwaysRun = true) public void afterClass() { safeClose(this.initClient); - safeClose(this.writeClient); - safeClose(this.readClient); } private String getDifferentLSNToken(String token, long lsnDifferent) throws Exception { diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/ConnectionStateListenerTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/ConnectionStateListenerTest.java index 4a994e86da47..01e7c1f86de4 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/ConnectionStateListenerTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/ConnectionStateListenerTest.java @@ -30,34 +30,43 @@ import com.azure.cosmos.implementation.routing.RegionalRoutingContext; import io.netty.handler.ssl.SslContext; import org.mockito.Mockito; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import java.io.IOException; import java.net.URI; -import java.net.URISyntaxException; import java.nio.channels.ClosedChannelException; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; -import java.util.Random; import java.util.UUID; -import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicInteger; import static com.azure.cosmos.implementation.TestUtils.mockDiagnosticsClientContext; import static org.assertj.core.api.Assertions.assertThat; public class ConnectionStateListenerTest { - private static final Logger logger = LoggerFactory.getLogger(ConnectionStateListenerTest.class); private static final AtomicInteger randomPort = new AtomicInteger(1000); - private static int port = 8082; - private static String serverAddressPrefix = "rntbd://localhost:"; - private static Random random = new Random(); + private static final int port = 8082; + private static final String serverAddressPrefix = "rntbd://localhost:"; + + private volatile AutoCloseable disableNettyLeakDetectionScope; + + @BeforeClass(groups = "unit") + public void beforeClass_DisableNettyLeakDetection() { + this.disableNettyLeakDetectionScope = CosmosNettyLeakDetectorFactory.createDisableLeakDetectionScope(); + } + + @AfterClass(groups = "unit", alwaysRun = true) + public void afterClass_ReactivateNettyLeakDetection() throws Exception { + if (this.disableNettyLeakDetectionScope != null) { + this.disableNettyLeakDetectionScope.close(); + } + } @DataProvider(name = "connectionStateListenerConfigProvider") public Object[][] connectionStateListenerConfigProvider() { diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/TcpServerMock/rntbd/ServerRntbdContextRequestDecoder.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/TcpServerMock/rntbd/ServerRntbdContextRequestDecoder.java index ff8dae6c33fb..a9ae3654115a 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/TcpServerMock/rntbd/ServerRntbdContextRequestDecoder.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/TcpServerMock/rntbd/ServerRntbdContextRequestDecoder.java @@ -41,13 +41,7 @@ public void channelRead(final ChannelHandlerContext context, final Object messag } } - // When bypassing the parent's decode logic, we must release the ByteBuf - // as context.fireChannelRead() doesn't automatically manage reference counts - try { - context.fireChannelRead(message); - } finally { - ReferenceCountUtil.safeRelease(message); - } + context.fireChannelRead(message); } /** diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/TcpServerMock/rntbd/ServerRntbdRequestDecoder.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/TcpServerMock/rntbd/ServerRntbdRequestDecoder.java index e67faae9edda..948a09f88a21 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/TcpServerMock/rntbd/ServerRntbdRequestDecoder.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/TcpServerMock/rntbd/ServerRntbdRequestDecoder.java @@ -35,13 +35,7 @@ public void channelRead(final ChannelHandlerContext context, final Object messag } } - // When bypassing the parent's decode logic, we must release the ByteBuf - // as context.fireChannelRead() doesn't automatically manage reference counts - try { - context.fireChannelRead(message); - } finally { - ReferenceCountUtil.safeRelease(message); - } + context.fireChannelRead(message); } /** diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdContextDecoder.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdContextDecoder.java index cf8724244535..13e953e47734 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdContextDecoder.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdContextDecoder.java @@ -6,6 +6,7 @@ import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.ByteToMessageDecoder; +import io.netty.util.ResourceLeakDetector; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -14,6 +15,8 @@ class RntbdContextDecoder extends ByteToMessageDecoder { private static final Logger logger = LoggerFactory.getLogger(RntbdContextDecoder.class); + private static final boolean leakDetectionDebuggingEnabled = ResourceLeakDetector.getLevel().ordinal() >= + ResourceLeakDetector.Level.ADVANCED.ordinal(); /** * Deserialize from an input {@link ByteBuf} to an {@link RntbdContext} instance @@ -27,6 +30,10 @@ class RntbdContextDecoder extends ByteToMessageDecoder { @Override protected void decode(final ChannelHandlerContext context, final ByteBuf in, final List out) { + if (leakDetectionDebuggingEnabled) { + in.touch("RntbdContextDecoder.decode: entry"); + } + if (RntbdFramer.canDecodeHead(in)) { Object result; @@ -35,14 +42,35 @@ protected void decode(final ChannelHandlerContext context, final ByteBuf in, fin final RntbdContext rntbdContext = RntbdContext.decode(in); context.fireUserEventTriggered(rntbdContext); result = rntbdContext; + + if (leakDetectionDebuggingEnabled) { + logger.info("{} RntbdContextDecoder: decoded RntbdContext successfully", context.channel()); + } } catch (RntbdContextException error) { context.fireUserEventTriggered(error); result = error; + + if (leakDetectionDebuggingEnabled) { + logger.info("{} RntbdContextDecoder: caught RntbdContextException", context.channel(), error); + } } finally { + if (leakDetectionDebuggingEnabled) { + in.touch("RntbdContextDecoder.decode: before discardReadBytes in finally block"); + } in.discardReadBytes(); } logger.debug("{} DECODE COMPLETE: {}", context.channel(), result); + } else if (leakDetectionDebuggingEnabled) { + logger.info("{} RntbdContextDecoder: cannot decode head yet, readableBytes={}", + context.channel(), in.readableBytes()); } } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + // BREADCRUMB: Track exceptions that might lead to leaked buffers + logger.warn("{} RntbdContextDecoder.exceptionCaught: {}", ctx.channel(), cause.getMessage(), cause); + super.exceptionCaught(ctx, cause); + } } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdContextRequestDecoder.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdContextRequestDecoder.java index c256fce6de65..ceec8d90ad46 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdContextRequestDecoder.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdContextRequestDecoder.java @@ -6,11 +6,19 @@ import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.ByteToMessageDecoder; +import io.netty.util.ResourceLeakDetector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.List; public class RntbdContextRequestDecoder extends ByteToMessageDecoder { + private static final Logger logger = LoggerFactory.getLogger(RntbdContextRequestDecoder.class); + + private static final boolean leakDetectionDebuggingEnabled = ResourceLeakDetector.getLevel().ordinal() >= + ResourceLeakDetector.Level.ADVANCED.ordinal(); + public RntbdContextRequestDecoder() { this.setSingleDecode(true); } @@ -28,14 +36,33 @@ public void channelRead(final ChannelHandlerContext context, final Object messag if (message instanceof ByteBuf) { final ByteBuf in = (ByteBuf)message; + // BREADCRUMB: Track buffer before reading operation type + if (leakDetectionDebuggingEnabled) { + in.touch("RntbdContextRequestDecoder.channelRead: before reading resourceOperationType"); + } + final int resourceOperationType = in.getInt(in.readerIndex() + Integer.BYTES); if (resourceOperationType == 0) { assert this.isSingleDecode(); + // BREADCRUMB: Going through normal decode path + if (leakDetectionDebuggingEnabled) { + in.touch("RntbdContextRequestDecoder.channelRead: passing to super.channelRead (resourceOperationType == 0)"); + } super.channelRead(context, message); return; } + + // BREADCRUMB: Bypassing decoder - this is a potential leak point if downstream doesn't release + if (leakDetectionDebuggingEnabled) { + in.touch("RntbdContextRequestDecoder.channelRead: bypassing decoder (resourceOperationType != 0)"); + } } + // BREADCRUMB: Message forwarded downstream - downstream MUST release it + if (leakDetectionDebuggingEnabled && message instanceof ByteBuf) { + ((ByteBuf) message).touch("RntbdContextRequestDecoder.channelRead: forwarding to next handler"); + } + context.fireChannelRead(message); } @@ -66,4 +93,11 @@ protected void decode(final ChannelHandlerContext context, final ByteBuf in, fin in.discardReadBytes(); out.add(request); } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + // BREADCRUMB: Track exceptions that might lead to leaked buffers + logger.warn("{} RntbdContextRequestDecoder.exceptionCaught: {}", ctx.channel(), cause.getMessage(), cause); + super.exceptionCaught(ctx, cause); + } } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdRequestDecoder.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdRequestDecoder.java index b924f2e97169..bab67b3dd231 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdRequestDecoder.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdRequestDecoder.java @@ -6,10 +6,19 @@ import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.ByteToMessageDecoder; +import io.netty.util.ResourceLeakDetector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.List; public final class RntbdRequestDecoder extends ByteToMessageDecoder { + + private static final Logger logger = LoggerFactory.getLogger(RntbdContextRequestDecoder.class); + + private static final boolean leakDetectionDebuggingEnabled = ResourceLeakDetector.getLevel().ordinal() >= + ResourceLeakDetector.Level.ADVANCED.ordinal(); + /** * Prepare for decoding an @{link RntbdRequest} or fire a channel readTree event to pass the input message along. * @@ -23,14 +32,32 @@ public void channelRead(final ChannelHandlerContext context, final Object messag if (message instanceof ByteBuf) { final ByteBuf in = (ByteBuf) message; + // BREADCRUMB: Track buffer before reading operation type + if (leakDetectionDebuggingEnabled) { + in.touch("RntbdRequestDecoder.channelRead: before reading resourceOperationType"); + } + final int resourceOperationType = in.getInt(in.readerIndex() + Integer.BYTES); if (resourceOperationType != 0) { + // BREADCRUMB: Going through normal decode path + if (leakDetectionDebuggingEnabled) { + in.touch("RntbdRequestDecoder.channelRead: passing to super.channelRead (resourceOperationType != 0)"); + } super.channelRead(context, message); return; } + + // BREADCRUMB: Bypassing decoder - this is a potential leak point if downstream doesn't release + if (leakDetectionDebuggingEnabled) { + in.touch("RntbdRequestDecoder.channelRead: bypassing decoder (resourceOperationType == 0)"); + } } + // BREADCRUMB: Message forwarded downstream - downstream MUST release it + if (leakDetectionDebuggingEnabled && message instanceof ByteBuf) { + ((ByteBuf) message).touch("RntbdRequestDecoder.channelRead: forwarding to next handler"); + } context.fireChannelRead(message); } @@ -65,4 +92,11 @@ protected void decode( in.discardReadBytes(); out.add(request); } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + // BREADCRUMB: Track exceptions that might lead to leaked buffers + logger.warn("{} RntbdRequestDecoder.exceptionCaught: {}", ctx.channel(), cause.getMessage(), cause); + super.exceptionCaught(ctx, cause); + } } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdResponseDecoder.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdResponseDecoder.java index 57c1c9a1a13a..8ae5da0aa315 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdResponseDecoder.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdResponseDecoder.java @@ -6,6 +6,7 @@ import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.ByteToMessageDecoder; +import io.netty.util.ResourceLeakDetector; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -18,6 +19,9 @@ public final class RntbdResponseDecoder extends ByteToMessageDecoder { private static final Logger logger = LoggerFactory.getLogger(RntbdResponseDecoder.class); private static final AtomicReference decodeStartTime = new AtomicReference<>(); + private static final boolean leakDetectionDebuggingEnabled = ResourceLeakDetector.getLevel().ordinal() >= + ResourceLeakDetector.Level.ADVANCED.ordinal(); + /** * Deserialize from an input {@link ByteBuf} to an {@link RntbdResponse} instance. *

@@ -32,6 +36,11 @@ protected void decode(final ChannelHandlerContext context, final ByteBuf in, fin decodeStartTime.compareAndSet(null, Instant.now()); + // BREADCRUMB: Track buffer at decode entry + if (leakDetectionDebuggingEnabled) { + in.touch("RntbdResponseDecoder.decode: entry"); + } + if (RntbdFramer.canDecodeHead(in)) { final RntbdResponse response = RntbdResponse.decode(in); @@ -41,9 +50,34 @@ protected void decode(final ChannelHandlerContext context, final ByteBuf in, fin response.setDecodeStartTime(decodeStartTime.getAndSet(null)); logger.debug("{} DECODE COMPLETE: {}", context.channel(), response); + + // BREADCRUMB: Track buffer before discard + if (leakDetectionDebuggingEnabled) { + in.touch("RntbdResponseDecoder.decode: before discardReadBytes"); + } + in.discardReadBytes(); + + // BREADCRUMB: Track response before adding to output + if (leakDetectionDebuggingEnabled) { + response.touch("RntbdResponseDecoder.decode: before retain and adding to output"); + } + out.add(response.retain()); + } else if (leakDetectionDebuggingEnabled) { + logger.info("{} RntbdResponseDecoder: response is null, not enough data to decode yet", + context.channel()); } + } else if (leakDetectionDebuggingEnabled) { + logger.info("{} RntbdResponseDecoder: cannot decode head yet, readableBytes={}", + context.channel(), in.readableBytes()); } } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + // BREADCRUMB: Track exceptions that might lead to leaked buffers + logger.warn("{} RntbdResponseDecoder.exceptionCaught: {}", ctx.channel(), cause.getMessage(), cause); + super.exceptionCaught(ctx, cause); + } } From 33a680eb5f28523e4121441728529079e21937c3 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Fri, 14 Nov 2025 03:27:06 +0000 Subject: [PATCH 53/65] Test fixes and more breadcrumbs --- .../CosmosNettyLeakDetectorFactory.java | 7 + .../CosmosNettyLeakDetectorFactory.java | 7 + .../cosmos/rx/ClientRetryPolicyE2ETests.java | 57 ++- .../implementation/RxGatewayStoreModel.java | 352 ++++++++++-------- .../implementation/ThinClientStoreModel.java | 4 +- 5 files changed, 230 insertions(+), 197 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java index f19ef115fa4f..759b4089e6c8 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java @@ -305,6 +305,13 @@ private static final class DisableLeakDetectionScope implements AutoCloseable { @Override public void close() { synchronized (staticLock) { + CosmosNettyLeakDetectorFactory.resetIdentifiedLeaks(); + try { + Thread.sleep(10_000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + CosmosNettyLeakDetectorFactory.resetIdentifiedLeaks(); isLeakDetectionDisabled = false; logger.info("Leak detection enabled again."); } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java index a4f4dd7e5878..58d80e48e54e 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java @@ -305,6 +305,13 @@ private static final class DisableLeakDetectionScope implements AutoCloseable { @Override public void close() { synchronized (staticLock) { + CosmosNettyLeakDetectorFactory.resetIdentifiedLeaks(); + try { + Thread.sleep(10_000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + CosmosNettyLeakDetectorFactory.resetIdentifiedLeaks(); isLeakDetectionDisabled = false; logger.info("Leak detection enabled again."); } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ClientRetryPolicyE2ETests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ClientRetryPolicyE2ETests.java index 66e22a8fdbef..446ee82008b8 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ClientRetryPolicyE2ETests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ClientRetryPolicyE2ETests.java @@ -551,52 +551,49 @@ public void dataPlaneRequestHitsLeaseNotFoundInFirstPreferredRegion( .hitLimit(hitLimit) .build(); - CosmosAsyncClient testClient = getClientBuilder() + try (CosmosAsyncClient testClient = getClientBuilder() .preferredRegions(shouldUsePreferredRegionsOnClient ? this.preferredRegions : Collections.emptyList()) .directMode() // required to force a quorum read irrespective of account consistency level .readConsistencyStrategy(ReadConsistencyStrategy.LATEST_COMMITTED) - .buildAsyncClient(); + .buildAsyncClient()) { - CosmosAsyncContainer testContainer = getSharedSinglePartitionCosmosContainer(testClient); + CosmosAsyncContainer testContainer = getSharedSinglePartitionCosmosContainer(testClient); - try { + try { - testContainer.createItem(createdItem).block(); - - CosmosFaultInjectionHelper.configureFaultInjectionRules(testContainer, Arrays.asList(leaseNotFoundFaultRule)).block(); + testContainer.createItem(createdItem).block(); - CosmosDiagnostics cosmosDiagnostics - = this.performDocumentOperation(testContainer, operationType, createdItem, testItem -> new PartitionKey(testItem.getMypk()), isReadMany) - .block(); + CosmosFaultInjectionHelper.configureFaultInjectionRules(testContainer, Arrays.asList(leaseNotFoundFaultRule)).block(); - if (shouldRetryCrossRegion) { - assertThat(cosmosDiagnostics).isNotNull(); - assertThat(cosmosDiagnostics.getDiagnosticsContext()).isNotNull(); + CosmosDiagnostics cosmosDiagnostics + = this.performDocumentOperation(testContainer, operationType, createdItem, testItem -> new PartitionKey(testItem.getMypk()), isReadMany) + .block(); - CosmosDiagnosticsContext diagnosticsContext = cosmosDiagnostics.getDiagnosticsContext(); + if (shouldRetryCrossRegion) { + assertThat(cosmosDiagnostics).isNotNull(); + assertThat(cosmosDiagnostics.getDiagnosticsContext()).isNotNull(); - assertThat(diagnosticsContext.getContactedRegionNames().size()).isEqualTo(2); - assertThat(diagnosticsContext.getStatusCode()).isLessThan(HttpConstants.StatusCodes.BADREQUEST); - assertThat(diagnosticsContext.getDuration()).isLessThan(Duration.ofSeconds(5)); - } else { - assertThat(cosmosDiagnostics).isNotNull(); - assertThat(cosmosDiagnostics.getDiagnosticsContext()).isNotNull(); + CosmosDiagnosticsContext diagnosticsContext = cosmosDiagnostics.getDiagnosticsContext(); - CosmosDiagnosticsContext diagnosticsContext = cosmosDiagnostics.getDiagnosticsContext(); + assertThat(diagnosticsContext.getContactedRegionNames().size()).isEqualTo(2); + assertThat(diagnosticsContext.getStatusCode()).isLessThan(HttpConstants.StatusCodes.BADREQUEST); + assertThat(diagnosticsContext.getDuration()).isLessThan(Duration.ofSeconds(5)); + } else { + assertThat(cosmosDiagnostics).isNotNull(); + assertThat(cosmosDiagnostics.getDiagnosticsContext()).isNotNull(); - assertThat(diagnosticsContext.getContactedRegionNames().size()).isEqualTo(1); - assertThat(diagnosticsContext.getStatusCode()).isEqualTo(HttpConstants.StatusCodes.SERVICE_UNAVAILABLE); - assertThat(diagnosticsContext.getSubStatusCode()).isEqualTo(HttpConstants.SubStatusCodes.LEASE_NOT_FOUND); - assertThat(diagnosticsContext.getDuration()).isLessThan(Duration.ofSeconds(5)); - } + CosmosDiagnosticsContext diagnosticsContext = cosmosDiagnostics.getDiagnosticsContext(); - } finally { - leaseNotFoundFaultRule.disable(); + assertThat(diagnosticsContext.getContactedRegionNames().size()).isEqualTo(1); + assertThat(diagnosticsContext.getStatusCode()).isEqualTo(HttpConstants.StatusCodes.SERVICE_UNAVAILABLE); + assertThat(diagnosticsContext.getSubStatusCode()).isEqualTo(HttpConstants.SubStatusCodes.LEASE_NOT_FOUND); + assertThat(diagnosticsContext.getDuration()).isLessThan(Duration.ofSeconds(5)); + } - if (testClient != null) { + } finally { + leaseNotFoundFaultRule.disable(); cleanUpContainer(testContainer); - testClient.close(); } } } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java index 680b591d38c9..ae45cd841e1a 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java @@ -217,7 +217,7 @@ public StoreResponse unwrapToStoreResponse( int size; if ((size = retainedContent.readableBytes()) > 0) { if (leakDetectionDebuggingEnabled) { - retainedContent.touch(this); + retainedContent.touch("RxGatewayStoreModel before creating StoreResponse - refCnt: " + retainedContent.refCnt()); } return new StoreResponse( @@ -406,210 +406,232 @@ private Mono toDocumentServiceResponse(Mono { - - // header key/value pairs - HttpHeaders httpResponseHeaders = httpResponse.headers(); - int httpResponseStatus = httpResponse.statusCode(); - - Mono contentObservable = httpResponse - .body() - .switchIfEmpty(Mono.just(Unpooled.EMPTY_BUFFER)) - .map(bodyByteBuf -> leakDetectionDebuggingEnabled - ? bodyByteBuf.retain().touch(this) - : bodyByteBuf.retain()) - .publishOn(CosmosSchedulers.TRANSPORT_RESPONSE_BOUNDED_ELASTIC) - .doOnDiscard(ByteBuf.class, buf -> { - if (buf.refCnt() > 0) { - // there could be a race with the catch in the .map operator below - // so, use safeRelease - ReferenceCountUtil.safeRelease(buf); - } - }); + return httpResponseMono + .publishOn(CosmosSchedulers.TRANSPORT_RESPONSE_BOUNDED_ELASTIC) + .flatMap(httpResponse -> { - return contentObservable - .map(content -> { - if (leakDetectionDebuggingEnabled) { - content.touch(this); - } + // header key/value pairs + HttpHeaders httpResponseHeaders = httpResponse.headers(); + int httpResponseStatus = httpResponse.statusCode(); - try { - // Capture transport client request timeline - ReactorNettyRequestRecord reactorNettyRequestRecord = httpResponse.request().reactorNettyRequestRecord(); - if (reactorNettyRequestRecord != null) { - reactorNettyRequestRecord.setTimeCompleted(Instant.now()); + Mono contentObservable = httpResponse + .body() + .switchIfEmpty(Mono.just(Unpooled.EMPTY_BUFFER)) + .map(bodyByteBuf -> { + if (leakDetectionDebuggingEnabled) { + bodyByteBuf.touch("RxGatewayStoreModel - before retain - refCnt: " + bodyByteBuf.refCnt()); } - StoreResponse rsp = request - .getEffectiveHttpTransportSerializer(this) - .unwrapToStoreResponse(httpRequest.uri().toString(), request, httpResponseStatus, httpResponseHeaders, content); - - if (reactorNettyRequestRecord != null) { - rsp.setRequestTimeline(reactorNettyRequestRecord.takeTimelineSnapshot()); - - if (this.gatewayServerErrorInjector != null) { - // only configure when fault injection is used - rsp.setFaultInjectionRuleId( - request - .faultInjectionRequestContext - .getFaultInjectionRuleId(reactorNettyRequestRecord.getTransportRequestId())); + // if not empty this is the aggregate result - for which ownership was already transferred + ByteBuf retainedContent = bodyByteBuf.retain(); + if (leakDetectionDebuggingEnabled) { + retainedContent.touch("RxGatewayStoreModel - after retain - refCnt: " + retainedContent.refCnt()); + } - rsp.setFaultInjectionRuleEvaluationResults( - request - .faultInjectionRequestContext - .getFaultInjectionRuleEvaluationResults(reactorNettyRequestRecord.getTransportRequestId())); + return retainedContent; + }) + .publishOn(CosmosSchedulers.TRANSPORT_RESPONSE_BOUNDED_ELASTIC) + .doOnDiscard(ByteBuf.class, buf -> { + if (buf.refCnt() > 0) { + if (leakDetectionDebuggingEnabled) { + buf.touch("RxGatewayStoreModel - doOnDiscard - begin - refCnt: " + buf.refCnt()); } - } - if (request.requestContext.cosmosDiagnostics != null) { - BridgeInternal.recordGatewayResponse(request.requestContext.cosmosDiagnostics, request, rsp, globalEndpointManager); + // there could be a race with the catch in the .map operator below + // so, use safeRelease + ReferenceCountUtil.safeRelease(buf); } + }); - return rsp; - } catch (Throwable t) { - if (content.refCnt() > 0) { - // Unwrap failed before StoreResponse took ownership -> release our retain - // there could be a race with the doOnDiscard above - so, use safeRelease - ReferenceCountUtil.safeRelease(content); + return contentObservable + .map(content -> { + if (leakDetectionDebuggingEnabled) { + content.touch("RxGatewayStoreModel - before capturing transport timeline - refCnt: " + content.refCnt()); } - throw t; - } - }) - .single(); - - }).map(rsp -> { - RxDocumentServiceResponse rxDocumentServiceResponse; - if (httpRequest.reactorNettyRequestRecord() != null) { - rxDocumentServiceResponse = - new RxDocumentServiceResponse(this.clientContext, rsp, - httpRequest.reactorNettyRequestRecord().takeTimelineSnapshot()); + try { + // Capture transport client request timeline + ReactorNettyRequestRecord reactorNettyRequestRecord = httpResponse.request().reactorNettyRequestRecord(); + if (reactorNettyRequestRecord != null) { + reactorNettyRequestRecord.setTimeCompleted(Instant.now()); + } - } else { - rxDocumentServiceResponse = - new RxDocumentServiceResponse(this.clientContext, rsp); - } - rxDocumentServiceResponse.setCosmosDiagnostics(request.requestContext.cosmosDiagnostics); - return rxDocumentServiceResponse; - }).onErrorResume(throwable -> { - Throwable unwrappedException = reactor.core.Exceptions.unwrap(throwable); - if (!(unwrappedException instanceof Exception)) { - // fatal error - logger.error("Unexpected failure " + unwrappedException.getMessage(), unwrappedException); - return Mono.error(unwrappedException); - } + if (leakDetectionDebuggingEnabled) { + content.touch("RxGatewayStoreModel - before creating StoreResponse - refCnt: " + content.refCnt()); + } + StoreResponse rsp = request + .getEffectiveHttpTransportSerializer(this) + .unwrapToStoreResponse(httpRequest.uri().toString(), request, httpResponseStatus, httpResponseHeaders, content); + + if (reactorNettyRequestRecord != null) { + rsp.setRequestTimeline(reactorNettyRequestRecord.takeTimelineSnapshot()); + + if (this.gatewayServerErrorInjector != null) { + // only configure when fault injection is used + rsp.setFaultInjectionRuleId( + request + .faultInjectionRequestContext + .getFaultInjectionRuleId(reactorNettyRequestRecord.getTransportRequestId())); + + rsp.setFaultInjectionRuleEvaluationResults( + request + .faultInjectionRequestContext + .getFaultInjectionRuleEvaluationResults(reactorNettyRequestRecord.getTransportRequestId())); + } + } - Exception exception = (Exception) unwrappedException; - CosmosException dce; - if (!(exception instanceof CosmosException)) { - int statusCode = 0; - if (WebExceptionUtility.isNetworkFailure(exception)) { + if (request.requestContext.cosmosDiagnostics != null) { + BridgeInternal.recordGatewayResponse(request.requestContext.cosmosDiagnostics, request, rsp, globalEndpointManager); + } - // wrap in CosmosException - logger.error("Network failure", exception); + return rsp; + } catch (Throwable t) { + if (content.refCnt() > 0) { + if (leakDetectionDebuggingEnabled) { + content.touch("RxGatewayStoreModel -exception creating StoreResponse - refCnt: " + content.refCnt()); + } + // Unwrap failed before StoreResponse took ownership -> release our retain + // there could be a race with the doOnDiscard above - so, use safeRelease + ReferenceCountUtil.safeRelease(content); + } - if (WebExceptionUtility.isReadTimeoutException(exception)) { - statusCode = HttpConstants.StatusCodes.REQUEST_TIMEOUT; - } else { - statusCode = HttpConstants.StatusCodes.SERVICE_UNAVAILABLE; - } - } + throw t; + } + }) + .single(); - dce = BridgeInternal.createCosmosException(request.requestContext.resourcePhysicalAddress, statusCode, exception); - BridgeInternal.setRequestHeaders(dce, request.getHeaders()); - } else { - logger.error("Non-network failure", exception); - dce = (CosmosException) exception; - } + }).map(rsp -> { + RxDocumentServiceResponse rxDocumentServiceResponse; + if (httpRequest.reactorNettyRequestRecord() != null) { + rxDocumentServiceResponse = + new RxDocumentServiceResponse(this.clientContext, rsp, + httpRequest.reactorNettyRequestRecord().takeTimelineSnapshot()); - if (WebExceptionUtility.isNetworkFailure(dce)) { - if (WebExceptionUtility.isReadTimeoutException(dce)) { - BridgeInternal.setSubStatusCode(dce, HttpConstants.SubStatusCodes.GATEWAY_ENDPOINT_READ_TIMEOUT); } else { - BridgeInternal.setSubStatusCode(dce, HttpConstants.SubStatusCodes.GATEWAY_ENDPOINT_UNAVAILABLE); + rxDocumentServiceResponse = + new RxDocumentServiceResponse(this.clientContext, rsp); } - } - - ImplementationBridgeHelpers - .CosmosExceptionHelper - .getCosmosExceptionAccessor() - .setRequestUri(dce, Uri.create(httpRequest.uri().toString())); - - if (request.requestContext.cosmosDiagnostics != null) { - if (httpRequest.reactorNettyRequestRecord() != null) { - ReactorNettyRequestRecord reactorNettyRequestRecord = httpRequest.reactorNettyRequestRecord(); - BridgeInternal.setRequestTimeline(dce, reactorNettyRequestRecord.takeTimelineSnapshot()); - - ImplementationBridgeHelpers - .CosmosExceptionHelper - .getCosmosExceptionAccessor() - .setFaultInjectionRuleId( - dce, - request.faultInjectionRequestContext - .getFaultInjectionRuleId(reactorNettyRequestRecord.getTransportRequestId())); - - ImplementationBridgeHelpers - .CosmosExceptionHelper - .getCosmosExceptionAccessor() - .setFaultInjectionEvaluationResults( - dce, - request.faultInjectionRequestContext - .getFaultInjectionRuleEvaluationResults(reactorNettyRequestRecord.getTransportRequestId())); + rxDocumentServiceResponse.setCosmosDiagnostics(request.requestContext.cosmosDiagnostics); + return rxDocumentServiceResponse; + }).onErrorResume(throwable -> { + Throwable unwrappedException = reactor.core.Exceptions.unwrap(throwable); + if (!(unwrappedException instanceof Exception)) { + // fatal error + logger.error("Unexpected failure " + unwrappedException.getMessage(), unwrappedException); + return Mono.error(unwrappedException); } - BridgeInternal.recordGatewayResponse(request.requestContext.cosmosDiagnostics, request, dce, globalEndpointManager); - } - - return Mono.error(dce); - }).doFinally(signalType -> { - - if (signalType != SignalType.CANCEL) { - return; - } - - if (httpRequest.reactorNettyRequestRecord() != null) { + Exception exception = (Exception) unwrappedException; + CosmosException dce; + if (!(exception instanceof CosmosException)) { + int statusCode = 0; + if (WebExceptionUtility.isNetworkFailure(exception)) { - OperationCancelledException oce = new OperationCancelledException("", httpRequest.uri()); + // wrap in CosmosException + logger.error("Network failure", exception); - ReactorNettyRequestRecord reactorNettyRequestRecord = httpRequest.reactorNettyRequestRecord(); - - RequestTimeline requestTimeline = reactorNettyRequestRecord.takeTimelineSnapshot(); - long transportRequestId = reactorNettyRequestRecord.getTransportRequestId(); - - GatewayRequestTimelineContext gatewayRequestTimelineContext = new GatewayRequestTimelineContext(requestTimeline, transportRequestId); - - request.requestContext.cancelledGatewayRequestTimelineContexts.add(gatewayRequestTimelineContext); + if (WebExceptionUtility.isReadTimeoutException(exception)) { + statusCode = HttpConstants.StatusCodes.REQUEST_TIMEOUT; + } else { + statusCode = HttpConstants.StatusCodes.SERVICE_UNAVAILABLE; + } + } - if (request.requestContext.getCrossRegionAvailabilityContext() != null) { + dce = BridgeInternal.createCosmosException(request.requestContext.resourcePhysicalAddress, statusCode, exception); + BridgeInternal.setRequestHeaders(dce, request.getHeaders()); + } else { + logger.error("Non-network failure", exception); + dce = (CosmosException) exception; + } - CrossRegionAvailabilityContextForRxDocumentServiceRequest availabilityStrategyContextForReq = - request.requestContext.getCrossRegionAvailabilityContext(); + if (WebExceptionUtility.isNetworkFailure(dce)) { + if (WebExceptionUtility.isReadTimeoutException(dce)) { + BridgeInternal.setSubStatusCode(dce, HttpConstants.SubStatusCodes.GATEWAY_ENDPOINT_READ_TIMEOUT); + } else { + BridgeInternal.setSubStatusCode(dce, HttpConstants.SubStatusCodes.GATEWAY_ENDPOINT_UNAVAILABLE); + } + } - if (availabilityStrategyContextForReq.getAvailabilityStrategyContext().isAvailabilityStrategyEnabled() && !availabilityStrategyContextForReq.getAvailabilityStrategyContext().isHedgedRequest()) { + ImplementationBridgeHelpers + .CosmosExceptionHelper + .getCosmosExceptionAccessor() + .setRequestUri(dce, Uri.create(httpRequest.uri().toString())); - BridgeInternal.setRequestTimeline(oce, reactorNettyRequestRecord.takeTimelineSnapshot()); + if (request.requestContext.cosmosDiagnostics != null) { + if (httpRequest.reactorNettyRequestRecord() != null) { + ReactorNettyRequestRecord reactorNettyRequestRecord = httpRequest.reactorNettyRequestRecord(); + BridgeInternal.setRequestTimeline(dce, reactorNettyRequestRecord.takeTimelineSnapshot()); ImplementationBridgeHelpers .CosmosExceptionHelper .getCosmosExceptionAccessor() .setFaultInjectionRuleId( - oce, + dce, request.faultInjectionRequestContext - .getFaultInjectionRuleId(transportRequestId)); + .getFaultInjectionRuleId(reactorNettyRequestRecord.getTransportRequestId())); ImplementationBridgeHelpers .CosmosExceptionHelper .getCosmosExceptionAccessor() .setFaultInjectionEvaluationResults( - oce, + dce, request.faultInjectionRequestContext - .getFaultInjectionRuleEvaluationResults(transportRequestId)); + .getFaultInjectionRuleEvaluationResults(reactorNettyRequestRecord.getTransportRequestId())); + } + + BridgeInternal.recordGatewayResponse(request.requestContext.cosmosDiagnostics, request, dce, globalEndpointManager); + } + + return Mono.error(dce); + }).doFinally(signalType -> { + + if (signalType != SignalType.CANCEL) { + return; + } - BridgeInternal.recordGatewayResponse(request.requestContext.cosmosDiagnostics, request, oce, globalEndpointManager); + if (httpRequest.reactorNettyRequestRecord() != null) { + + OperationCancelledException oce = new OperationCancelledException("", httpRequest.uri()); + + ReactorNettyRequestRecord reactorNettyRequestRecord = httpRequest.reactorNettyRequestRecord(); + + RequestTimeline requestTimeline = reactorNettyRequestRecord.takeTimelineSnapshot(); + long transportRequestId = reactorNettyRequestRecord.getTransportRequestId(); + + GatewayRequestTimelineContext gatewayRequestTimelineContext = new GatewayRequestTimelineContext(requestTimeline, transportRequestId); + + request.requestContext.cancelledGatewayRequestTimelineContexts.add(gatewayRequestTimelineContext); + + if (request.requestContext.getCrossRegionAvailabilityContext() != null) { + + CrossRegionAvailabilityContextForRxDocumentServiceRequest availabilityStrategyContextForReq = + request.requestContext.getCrossRegionAvailabilityContext(); + + if (availabilityStrategyContextForReq.getAvailabilityStrategyContext().isAvailabilityStrategyEnabled() && !availabilityStrategyContextForReq.getAvailabilityStrategyContext().isHedgedRequest()) { + + BridgeInternal.setRequestTimeline(oce, reactorNettyRequestRecord.takeTimelineSnapshot()); + + ImplementationBridgeHelpers + .CosmosExceptionHelper + .getCosmosExceptionAccessor() + .setFaultInjectionRuleId( + oce, + request.faultInjectionRequestContext + .getFaultInjectionRuleId(transportRequestId)); + + ImplementationBridgeHelpers + .CosmosExceptionHelper + .getCosmosExceptionAccessor() + .setFaultInjectionEvaluationResults( + oce, + request.faultInjectionRequestContext + .getFaultInjectionRuleEvaluationResults(transportRequestId)); + + BridgeInternal.recordGatewayResponse(request.requestContext.cosmosDiagnostics, request, oce, globalEndpointManager); + } } } - } - }); + }); } private void validateOrThrow(RxDocumentServiceRequest request, @@ -624,11 +646,11 @@ private void validateOrThrow(RxDocumentServiceRequest request, ? status.reasonPhrase().replace(" ", "") : ""; - String body = retainedBodyAsByteBuf != null + String body = retainedBodyAsByteBuf.readableBytes() > 0 ? retainedBodyAsByteBuf.toString(StandardCharsets.UTF_8) : null; - retainedBodyAsByteBuf.release(); + ReferenceCountUtil.safeRelease(retainedBodyAsByteBuf); CosmosError cosmosError; cosmosError = (StringUtils.isNotEmpty(body)) ? new CosmosError(body) : new CosmosError(); diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ThinClientStoreModel.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ThinClientStoreModel.java index 74b314881bba..d32e5d901f18 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ThinClientStoreModel.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ThinClientStoreModel.java @@ -111,7 +111,7 @@ public StoreResponse unwrapToStoreResponse( } if (leakDetectionDebuggingEnabled) { - content.touch(this); + content.touch("ThinClientStoreModel.unwrapToStoreResponse - refCnt: " + content.refCnt()); } try { @@ -123,7 +123,7 @@ public StoreResponse unwrapToStoreResponse( ByteBuf payloadBuf = response.getContent(); if (payloadBuf != Unpooled.EMPTY_BUFFER && leakDetectionDebuggingEnabled) { - payloadBuf.touch(this); + payloadBuf.touch("ThinClientStoreModel.after RNTBD decoding - refCnt: " + payloadBuf.refCnt()); } try { From 02ee9f3d538477ce3ed1d070b4dc37d14dcbdfc1 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Fri, 14 Nov 2025 14:09:18 +0000 Subject: [PATCH 54/65] Test fixes --- .../cosmos/RetryContextOnDiagnosticTest.java | 4 +- .../com/azure/cosmos/rx/OfferQueryTest.java | 49 +++++++++++-------- .../implementation/RxGatewayStoreModel.java | 29 ++++++++--- .../directconnectivity/StoreResponse.java | 7 ++- .../http/ReactorNettyClient.java | 32 ++++++++++-- 5 files changed, 85 insertions(+), 36 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/RetryContextOnDiagnosticTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/RetryContextOnDiagnosticTest.java index b6190b00f116..b3a09b36d3f7 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/RetryContextOnDiagnosticTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/RetryContextOnDiagnosticTest.java @@ -98,12 +98,12 @@ public class RetryContextOnDiagnosticTest extends TestSuiteBase { private AddressSelector addressSelector; private volatile AutoCloseable disableNettyLeakDetectionScope; - @BeforeClass(groups = "unit") + @BeforeClass(groups = {"unit", "long-emulator"}) public void beforeClass_DisableNettyLeakDetection() { this.disableNettyLeakDetectionScope = CosmosNettyLeakDetectorFactory.createDisableLeakDetectionScope(); } - @AfterClass(groups = "unit", alwaysRun = true) + @AfterClass(groups = {"unit", "long-emulator"}, alwaysRun = true) public void afterClass_ReactivateNettyLeakDetection() throws Exception { if (this.disableNettyLeakDetectionScope != null) { this.disableNettyLeakDetectionScope.close(); diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OfferQueryTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OfferQueryTest.java index fb814862e6ea..a8743a1f6b9c 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OfferQueryTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OfferQueryTest.java @@ -153,30 +153,37 @@ public void queryCollections_NoResults() throws Exception { String query = "SELECT * from root r where r.id = '2'"; CosmosQueryRequestOptions options = new CosmosQueryRequestOptions(); - CosmosAsyncClient cosmosClient = new CosmosClientBuilder() + try (CosmosAsyncClient cosmosClient = new CosmosClientBuilder() .key(TestConfigurations.MASTER_KEY) .endpoint(TestConfigurations.HOST) - .buildAsyncClient(); - QueryFeedOperationState dummyState = new QueryFeedOperationState( - cosmosClient, - "SomeSpanName", - "SomeDBName", - "SomeContainerName", - ResourceType.Document, - OperationType.Query, - null, - options, - new CosmosPagedFluxOptions() - ); - Flux> queryObservable = client.queryCollections(getDatabaseLink(), query, dummyState); - - FeedResponseListValidator validator = new FeedResponseListValidator.Builder() - .containsExactly(new ArrayList<>()) - .numberOfPages(1) - .pageSatisfy(0, new FeedResponseValidator.Builder() + .buildAsyncClient()) { + + QueryFeedOperationState dummyState = new QueryFeedOperationState( + cosmosClient, + "SomeSpanName", + "SomeDBName", + "SomeContainerName", + ResourceType.Document, + OperationType.Query, + null, + options, + new CosmosPagedFluxOptions() + ); + + try { + Flux> queryObservable = client.queryCollections(getDatabaseLink(), query, dummyState); + + FeedResponseListValidator validator = new FeedResponseListValidator.Builder() + .containsExactly(new ArrayList<>()) + .numberOfPages(1) + .pageSatisfy(0, new FeedResponseValidator.Builder() .requestChargeGreaterThanOrEqualTo(1.0).build()) - .build(); - validateQuerySuccess(queryObservable, validator); + .build(); + validateQuerySuccess(queryObservable, validator); + } finally { + safeClose(dummyState); + } + } } @BeforeClass(groups = { "query" }, timeOut = SETUP_TIMEOUT) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java index ae45cd841e1a..ff116a3088f1 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java @@ -211,6 +211,12 @@ public StoreResponse unwrapToStoreResponse( retainedContent, "Argument 'retainedContent' must not be null - use empty ByteBuf when theres is no payload."); + if (leakDetectionDebuggingEnabled) { + retainedContent.touch( + "RxGatewayStoreModel.unwrapToStoreResponse before validate - refCnt: " + retainedContent.refCnt()); + logger.info("RxGatewayStoreModel.unwrapToStoreResponse before validate - refCnt: {}", retainedContent.refCnt()); + } + // If there is any error in the header response this throws exception validateOrThrow(request, HttpResponseStatus.valueOf(statusCode), headers, retainedContent); @@ -218,6 +224,7 @@ public StoreResponse unwrapToStoreResponse( if ((size = retainedContent.readableBytes()) > 0) { if (leakDetectionDebuggingEnabled) { retainedContent.touch("RxGatewayStoreModel before creating StoreResponse - refCnt: " + retainedContent.refCnt()); + logger.info("RxGatewayStoreModel before creating StoreResponse - refCnt: {}", retainedContent.refCnt()); } return new StoreResponse( @@ -419,34 +426,40 @@ private Mono toDocumentServiceResponse(Mono { if (leakDetectionDebuggingEnabled) { - bodyByteBuf.touch("RxGatewayStoreModel - before retain - refCnt: " + bodyByteBuf.refCnt()); + bodyByteBuf.touch("RxGatewayStoreModel - buffer after aggregate before retain - refCnt: " + bodyByteBuf.refCnt()); + logger.info("RxGatewayStoreModel - buffer after aggregate before retain - refCnt: {}", bodyByteBuf.refCnt()); + } + + if (bodyByteBuf != Unpooled.EMPTY_BUFFER) { + bodyByteBuf.retain(); // +1 for our downstream work } - // if not empty this is the aggregate result - for which ownership was already transferred - ByteBuf retainedContent = bodyByteBuf.retain(); if (leakDetectionDebuggingEnabled) { - retainedContent.touch("RxGatewayStoreModel - after retain - refCnt: " + retainedContent.refCnt()); + bodyByteBuf.touch("RxGatewayStoreModel - touch retained buffer - refCnt: " + bodyByteBuf.refCnt()); + logger.info("RxGatewayStoreModel - touch retained buffer - refCnt: {]", bodyByteBuf.refCnt()); } - return retainedContent; + return bodyByteBuf; }) - .publishOn(CosmosSchedulers.TRANSPORT_RESPONSE_BOUNDED_ELASTIC) .doOnDiscard(ByteBuf.class, buf -> { if (buf.refCnt() > 0) { if (leakDetectionDebuggingEnabled) { buf.touch("RxGatewayStoreModel - doOnDiscard - begin - refCnt: " + buf.refCnt()); + logger.info("RxGatewayStoreModel - doOnDiscard - begin - refCnt: {}", buf.refCnt()); } // there could be a race with the catch in the .map operator below // so, use safeRelease ReferenceCountUtil.safeRelease(buf); } - }); + }) + .publishOn(CosmosSchedulers.TRANSPORT_RESPONSE_BOUNDED_ELASTIC); return contentObservable .map(content -> { if (leakDetectionDebuggingEnabled) { content.touch("RxGatewayStoreModel - before capturing transport timeline - refCnt: " + content.refCnt()); + logger.info("RxGatewayStoreModel - before capturing transport timeline - refCnt: {}", content.refCnt()); } try { @@ -458,6 +471,7 @@ private Mono toDocumentServiceResponse(Mono toDocumentServiceResponse(Mono 0) { if (leakDetectionDebuggingEnabled) { content.touch("RxGatewayStoreModel -exception creating StoreResponse - refCnt: " + content.refCnt()); + logger.info("RxGatewayStoreModel -exception creating StoreResponse - refCnt: {}", content.refCnt()); } // Unwrap failed before StoreResponse took ownership -> release our retain // there could be a race with the doOnDiscard above - so, use safeRelease diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreResponse.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreResponse.java index a4a3cccd558c..e092886a8d08 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreResponse.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreResponse.java @@ -11,6 +11,7 @@ import com.azure.cosmos.implementation.directconnectivity.rntbd.RntbdEndpointStatistics; import com.fasterxml.jackson.databind.JsonNode; import io.netty.buffer.ByteBufInputStream; +import io.netty.util.IllegalReferenceCountException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -75,8 +76,10 @@ public StoreResponse( try { contentStream.close(); } catch (Throwable e) { - // Log as warning instead of debug to make ByteBuf leak issues more visible - logger.warn("Failed to close content stream. This may cause a Netty ByteBuf leak.", e); + if (!(e instanceof IllegalReferenceCountException)) { + // Log as warning instead of debug to make ByteBuf leak issues more visible + logger.warn("Failed to close content stream. This may cause a Netty ByteBuf leak.", e); + } } } } else { diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java index 488596e825d5..aec3eb243853 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java @@ -11,6 +11,8 @@ import io.netty.handler.codec.http.HttpMethod; import io.netty.handler.logging.LogLevel; import io.netty.resolver.DefaultAddressResolverGroup; +import io.netty.util.ReferenceCountUtil; +import io.netty.util.ResourceLeakDetector; import org.reactivestreams.Publisher; import org.reactivestreams.Subscription; import org.slf4j.Logger; @@ -39,7 +41,8 @@ * HttpClient that is implemented using reactor-netty. */ public class ReactorNettyClient implements HttpClient { - + private static final boolean leakDetectionDebuggingEnabled = ResourceLeakDetector.getLevel().ordinal() >= + ResourceLeakDetector.Level.ADVANCED.ordinal(); private static final String REACTOR_NETTY_REQUEST_RECORD_KEY = "reactorNettyRequestRecordKey"; private static final Logger logger = LoggerFactory.getLogger(ReactorNettyClient.class.getSimpleName()); @@ -348,7 +351,15 @@ public HttpHeaders headers() { public Mono body() { return ByteBufFlux .fromInbound( - bodyIntern().doOnDiscard(ByteBuf.class, io.netty.util.ReferenceCountUtil::safeRelease) + bodyIntern().doOnDiscard( + ByteBuf.class, + buf -> { + if (leakDetectionDebuggingEnabled && buf.refCnt() > 0) { + buf.touch("ReactorNettyHttpResponse.body - onDiscard - refCnt: " + buf.refCnt()); + logger.info("ReactorNettyHttpResponse.body - onDiscard - refCnt: {}", buf.refCnt()); + ReferenceCountUtil.safeRelease(buf); + } + }) ) .aggregate() .doOnSubscribe(this::updateSubscriptionState); @@ -400,8 +411,21 @@ private void releaseOnNotSubscribedResponse(ReactorNettyResponseState reactorNet if (logger.isDebugEnabled()) { logger.debug("Releasing body, not yet subscribed"); } - this.bodyIntern() - .doOnNext(io.netty.util.ReferenceCountUtil::safeRelease) + + if (leakDetectionDebuggingEnabled) { + logger.info("Releasing body, not yet subscribed"); + } + + body() + .map(buf -> { + if (leakDetectionDebuggingEnabled && buf.refCnt() > 0) { + buf.touch("ReactorNettyHttpResponse.releaseOnNotSubscribedResponse - refCnt: " + buf.refCnt()); + logger.info("ReactorNettyHttpResponse.releaseOnNotSubscribedResponse - refCnt: {}", buf.refCnt()); + ReferenceCountUtil.safeRelease(buf); + } + + return buf; + }) .subscribe(v -> {}, ex -> {}, () -> {}); } } From a54d1ba4e000869f4aed36ed6f9eafafb1a2609e Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Fri, 14 Nov 2025 22:49:36 +0000 Subject: [PATCH 55/65] Test fixes --- ...PerPartitionAutomaticFailoverE2ETests.java | 98 +++++++++++++------ .../cosmos/RetryContextOnDiagnosticTest.java | 80 ++++++++------- .../implementation/RxGatewayStoreModel.java | 3 +- 3 files changed, 114 insertions(+), 67 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/PerPartitionAutomaticFailoverE2ETests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/PerPartitionAutomaticFailoverE2ETests.java index c7f09c818050..92d21daf488b 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/PerPartitionAutomaticFailoverE2ETests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/PerPartitionAutomaticFailoverE2ETests.java @@ -69,9 +69,11 @@ import io.netty.buffer.ByteBufUtil; import io.netty.handler.codec.http.HttpMethod; import io.netty.handler.timeout.ReadTimeoutException; +import io.netty.util.ReferenceCountUtil; import org.assertj.core.api.Assertions; import org.mockito.Mockito; import org.testng.SkipException; +import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; import org.testng.annotations.Factory; @@ -95,6 +97,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; +import java.util.function.Supplier; import static org.assertj.core.api.Assertions.assertThat; @@ -212,6 +215,7 @@ */ public class PerPartitionAutomaticFailoverE2ETests extends TestSuiteBase { + private CosmosAsyncClient sharedClient; private CosmosAsyncDatabase sharedDatabase; private CosmosAsyncContainer sharedSinglePartitionContainer; private AccountLevelLocationContext accountLevelLocationReadableLocationContext; @@ -462,10 +466,10 @@ public Object[][] ppafDynamicEnablement503Only() { @BeforeClass(groups = {"multi-region"}) public void beforeClass() { - CosmosAsyncClient cosmosAsyncClient = getClientBuilder().buildAsyncClient(); + this.sharedClient = getClientBuilder().buildAsyncClient(); - this.sharedDatabase = getSharedCosmosDatabase(cosmosAsyncClient); - this.sharedSinglePartitionContainer = getSharedSinglePartitionCosmosContainer(cosmosAsyncClient); + this.sharedDatabase = getSharedCosmosDatabase(this.sharedClient); + this.sharedSinglePartitionContainer = getSharedSinglePartitionCosmosContainer(this.sharedClient); ONLY_GATEWAY_MODE.add(ConnectionMode.GATEWAY); ONLY_DIRECT_MODE.add(ConnectionMode.DIRECT); @@ -473,13 +477,20 @@ public void beforeClass() { ALL_CONNECTION_MODES.add(ConnectionMode.DIRECT); ALL_CONNECTION_MODES.add(ConnectionMode.GATEWAY); - RxDocumentClientImpl rxDocumentClient = (RxDocumentClientImpl) ReflectionUtils.getAsyncDocumentClient(cosmosAsyncClient); + RxDocumentClientImpl rxDocumentClient = (RxDocumentClientImpl) ReflectionUtils.getAsyncDocumentClient(this.sharedClient); GlobalEndpointManager globalEndpointManager = ReflectionUtils.getGlobalEndpointManager(rxDocumentClient); DatabaseAccount databaseAccountSnapshot = globalEndpointManager.getLatestDatabaseAccount(); this.accountLevelLocationReadableLocationContext = getAccountLevelLocationContext(databaseAccountSnapshot, false); } + @AfterClass(groups = {"multi-region"}) + public void afterClass() throws InterruptedException { + safeClose(this.sharedClient); + System.gc(); + Thread.sleep(10_000); + } + @DataProvider(name = "ppafTestConfigsWithWriteOps") public Object[][] ppafTestConfigsWithWriteOps() { @@ -2821,32 +2832,49 @@ public HttpHeaders headers() { @Override public Mono body() { - try { - - if (resourceType == ResourceType.DatabaseAccount) { - return Mono.just(ByteBufUtil.writeUtf8(ByteBufAllocator.DEFAULT, databaseAccount.toJson())); - } - - if (operationType == OperationType.Batch) { - FakeBatchResponse fakeBatchResponse = new FakeBatchResponse(); - - fakeBatchResponse - .seteTag("1") - .setStatusCode(HttpConstants.StatusCodes.OK) - .setSubStatusCode(HttpConstants.SubStatusCodes.UNKNOWN) - .setRequestCharge(1.0d) - .setResourceBody(getTestPojoObject()) - .setRetryAfterMilliseconds("1"); - - return Mono.just(ByteBufUtil.writeUtf8(ByteBufAllocator.DEFAULT, - OBJECT_MAPPER.writeValueAsString(Arrays.asList(fakeBatchResponse)))); - } + if (resourceType == ResourceType.DatabaseAccount) { + return createAutoReleasingMono( + () -> ByteBufUtil.writeUtf8( + ByteBufAllocator.DEFAULT, + databaseAccount.toJson() + )); + } - return Mono.just(ByteBufUtil.writeUtf8(ByteBufAllocator.DEFAULT, - OBJECT_MAPPER.writeValueAsString(testPojo))); - } catch (JsonProcessingException e) { - return Mono.error(e); + if (operationType == OperationType.Batch) { + FakeBatchResponse fakeBatchResponse = new FakeBatchResponse(); + + fakeBatchResponse + .seteTag("1") + .setStatusCode(HttpConstants.StatusCodes.OK) + .setSubStatusCode(HttpConstants.SubStatusCodes.UNKNOWN) + .setRequestCharge(1.0d) + .setResourceBody(getTestPojoObject()) + .setRetryAfterMilliseconds("1"); + + return createAutoReleasingMono( + () -> { + try { + return ByteBufUtil.writeUtf8( + ByteBufAllocator.DEFAULT, + OBJECT_MAPPER.writeValueAsString(Arrays.asList(fakeBatchResponse)) + ); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + }); } + + return createAutoReleasingMono( + () -> { + try { + return ByteBufUtil.writeUtf8( + ByteBufAllocator.DEFAULT, + OBJECT_MAPPER.writeValueAsString(testPojo) + ); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + }); } @Override @@ -2884,4 +2912,18 @@ public Mono bodyAsString() { return httpResponse; } } + + private Mono createAutoReleasingMono(Supplier bufferSupplier) { + final AtomicReference output = new AtomicReference<>(null); + + return Mono + .fromCallable(() -> { + ByteBuf buf = bufferSupplier.get(); + output.set(buf); + + return buf; + }) + .doOnDiscard(ByteBuf.class, ReferenceCountUtil::safeRelease) + .doFinally(signalType -> ReferenceCountUtil.safeRelease(output.get())); + } } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/RetryContextOnDiagnosticTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/RetryContextOnDiagnosticTest.java index b3a09b36d3f7..ee59f0ac723e 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/RetryContextOnDiagnosticTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/RetryContextOnDiagnosticTest.java @@ -58,14 +58,13 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufAllocator; -import io.netty.buffer.ByteBufInputStream; import io.netty.buffer.ByteBufUtil; import io.netty.handler.codec.http.HttpMethod; +import io.netty.util.ReferenceCountUtil; import io.reactivex.subscribers.TestSubscriber; import org.mockito.ArgumentMatchers; import org.mockito.Mockito; import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; import reactor.core.publisher.Mono; @@ -73,7 +72,6 @@ import java.net.URISyntaxException; import java.time.Duration; import java.util.Arrays; -import java.util.HashMap; import java.util.Iterator; import java.util.Optional; import java.util.UUID; @@ -81,10 +79,10 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import static com.azure.cosmos.implementation.TestUtils.mockDiagnosticsClientContext; -import static com.azure.cosmos.implementation.Utils.getUTF8BytesOrNull; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Fail.fail; @@ -96,18 +94,11 @@ public class RetryContextOnDiagnosticTest extends TestSuiteBase { private IRetryPolicy retryPolicy; private RxDocumentServiceRequest serviceRequest; private AddressSelector addressSelector; - private volatile AutoCloseable disableNettyLeakDetectionScope; - - @BeforeClass(groups = {"unit", "long-emulator"}) - public void beforeClass_DisableNettyLeakDetection() { - this.disableNettyLeakDetectionScope = CosmosNettyLeakDetectorFactory.createDisableLeakDetectionScope(); - } @AfterClass(groups = {"unit", "long-emulator"}, alwaysRun = true) public void afterClass_ReactivateNettyLeakDetection() throws Exception { - if (this.disableNettyLeakDetectionScope != null) { - this.disableNettyLeakDetectionScope.close(); - } + System.gc(); + Thread.sleep(10_000); } @Test(groups = {"unit"}, timeOut = TIMEOUT * 2) @@ -869,6 +860,7 @@ public void throttlingExceptionGatewayModeScenario() { .key(TestConfigurations.MASTER_KEY) .gatewayMode() .buildClient(); + HttpClient mockHttpClient = null; try { CosmosAsyncContainer cosmosAsyncContainer = getSharedMultiPartitionCosmosContainer(cosmosClient.asyncClient()); @@ -893,18 +885,18 @@ public void throttlingExceptionGatewayModeScenario() { options.setPartitionKey(new PartitionKey(testPojo.getMypk())); options.setReadConsistencyStrategy(ReadConsistencyStrategy.EVENTUAL); Iterator> iterator = cosmosContainer.queryItems(query, - options, InternalObjectNode.class) - .iterableByPage(1) - .iterator(); + options, InternalObjectNode.class) + .iterableByPage(1) + .iterator(); FeedResponse feedResponse = iterator.next(); // Query Plan Caching end - HttpClient mockHttpClient = Mockito.mock(HttpClient.class); + mockHttpClient = Mockito.mock(HttpClient.class); CosmosException throttlingException = new CosmosException(429, "Throttling Test"); Mockito.when(mockHttpClient.send(Mockito.any(HttpRequest.class), Mockito.any(Duration.class))) - .thenReturn(Mono.error(throttlingException), Mono.error(throttlingException), - Mono.just(createResponse((201)))); + .thenReturn(Mono.error(throttlingException), Mono.error(throttlingException), + Mono.just(createResponse((201)))); ReflectionUtils.setGatewayHttpClient(rxGatewayStoreModel, mockHttpClient); CosmosItemResponse createItemResponse = cosmosContainer.createItem(testPojo, @@ -914,10 +906,11 @@ public void throttlingExceptionGatewayModeScenario() { assertThat(retryContext.getRetryCount()).isEqualTo(2); assertThat(retryContext.getStatusAndSubStatusCodes().get(0)[0]).isEqualTo(429); + mockHttpClient.shutdown(); mockHttpClient = Mockito.mock(HttpClient.class); Mockito.when(mockHttpClient.send(Mockito.any(HttpRequest.class), Mockito.any(Duration.class))) - .thenReturn(Mono.error(throttlingException), Mono.error(throttlingException), - Mono.just(createResponse((201)))); + .thenReturn(Mono.error(throttlingException), Mono.error(throttlingException), + Mono.just(createResponse((201)))); ReflectionUtils.setGatewayHttpClient(rxGatewayStoreModel, mockHttpClient); CosmosItemRequestOptions requestOptions = new CosmosItemRequestOptions(); @@ -930,30 +923,34 @@ public void throttlingExceptionGatewayModeScenario() { assertThat(retryContext.getRetryCount()).isEqualTo(2); assertThat(retryContext.getStatusAndSubStatusCodes().get(0)[0]).isEqualTo(429); + mockHttpClient.shutdown(); mockHttpClient = Mockito.mock(HttpClient.class); Mockito.when(mockHttpClient.send(Mockito.any(HttpRequest.class), Mockito.any(Duration.class))) - .thenReturn(Mono.error(throttlingException), Mono.error(throttlingException), - Mono.just(createResponse((201)))); + .thenReturn(Mono.error(throttlingException), Mono.error(throttlingException), + Mono.just(createResponse((201)))); ReflectionUtils.setGatewayHttpClient(rxGatewayStoreModel, mockHttpClient); options.setConsistencyLevel(ConsistencyLevel.EVENTUAL); iterator = cosmosContainer.queryItems(query, - options, InternalObjectNode.class) - .iterableByPage() - .iterator(); + options, InternalObjectNode.class) + .iterableByPage() + .iterator(); feedResponse = iterator.next(); Optional first = feedResponse.getCosmosDiagnostics() - .getFeedResponseDiagnostics() - .getClientSideRequestStatistics() - .stream() - .filter(context -> context.getRetryContext().getRetryCount() == 2 - && context.getRetryContext().getStatusAndSubStatusCodes().get(0)[0] == 429) - .findFirst(); + .getFeedResponseDiagnostics() + .getClientSideRequestStatistics() + .stream() + .filter(context -> context.getRetryContext().getRetryCount() == 2 + && context.getRetryContext().getStatusAndSubStatusCodes().get(0)[0] == 429) + .findFirst(); assertThat(first.isPresent()).isTrue(); System.setProperty("COSMOS.QUERYPLAN_CACHING_ENABLED", "false"); } finally { safeCloseSyncClient(cosmosClient); + if (mockHttpClient != null) { + mockHttpClient.shutdown(); + } } } @@ -1067,12 +1064,19 @@ public HttpHeaders headers() { @Override public Mono body() { - try { - return Mono.just(ByteBufUtil.writeUtf8(ByteBufAllocator.DEFAULT, - OBJECT_MAPPER.writeValueAsString(getTestPojoObject()))); - } catch (JsonProcessingException e) { - return Mono.error(e); - } + final AtomicReference output = new AtomicReference<>(null); + + return Mono + .fromCallable(() -> { + ByteBuf buf = ByteBufUtil.writeUtf8( + ByteBufAllocator.DEFAULT, + OBJECT_MAPPER.writeValueAsString(getTestPojoObject())); + output.set(buf); + + return buf; + }) + .doOnDiscard(ByteBuf.class, ReferenceCountUtil::safeRelease) + .doFinally(signalType -> ReferenceCountUtil.safeRelease(output.get())); } @Override diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java index ff116a3088f1..1f28fc686c91 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java @@ -431,7 +431,8 @@ private Mono toDocumentServiceResponse(Mono Date: Sat, 15 Nov 2025 00:32:25 +0000 Subject: [PATCH 56/65] Test fixes --- .../ClientRetryPolicyE2ETestsWithGatewayV2.java | 11 +++++++---- .../implementation/RxGatewayStoreModel.java | 11 +++++++++++ .../http/Http2ResponseHeaderCleanerHandler.java | 4 ++-- .../implementation/http/ReactorNettyClient.java | 16 ++++++++++------ 4 files changed, 30 insertions(+), 12 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ClientRetryPolicyE2ETestsWithGatewayV2.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ClientRetryPolicyE2ETestsWithGatewayV2.java index 6c8f80e66a75..66d9621cbc1f 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ClientRetryPolicyE2ETestsWithGatewayV2.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/ClientRetryPolicyE2ETestsWithGatewayV2.java @@ -81,11 +81,14 @@ public ClientRetryPolicyE2ETestsWithGatewayV2(CosmosClientBuilder clientBuilder) @BeforeClass(groups = {"fi-thinclient-multi-region", "fi-thinclient-multi-master"}, timeOut = TIMEOUT) public void beforeClass() { - CosmosAsyncClient dummy = getClientBuilder().buildAsyncClient(); - AsyncDocumentClient asyncDocumentClient = BridgeInternal.getContextClient(dummy); - GlobalEndpointManager globalEndpointManager = asyncDocumentClient.getGlobalEndpointManager(); + DatabaseAccount databaseAccount; - DatabaseAccount databaseAccount = globalEndpointManager.getLatestDatabaseAccount(); + try (CosmosAsyncClient dummy = getClientBuilder().buildAsyncClient()) { + AsyncDocumentClient asyncDocumentClient = BridgeInternal.getContextClient(dummy); + GlobalEndpointManager globalEndpointManager = asyncDocumentClient.getGlobalEndpointManager(); + + databaseAccount = globalEndpointManager.getLatestDatabaseAccount(); + } this.accountLevelReadableLocationContext = getAccountLevelLocationContext(databaseAccount, false); diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java index 1f28fc686c91..cdb4c29cb486 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java @@ -514,6 +514,17 @@ private Mono toDocumentServiceResponse(Mono { + // This handles the case where the retained buffer is discarded after the map operation + // but before unwrapToStoreResponse takes ownership (e.g., during cancellation) + if (buf.refCnt() > 0) { + if (leakDetectionDebuggingEnabled) { + buf.touch("RxGatewayStoreModel - doOnDiscard after map - refCnt: " + buf.refCnt()); + logger.info("RxGatewayStoreModel - doOnDiscard after map - refCnt: {}", buf.refCnt()); + } + ReferenceCountUtil.safeRelease(buf); + } + }) .single(); }).map(rsp -> { diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2ResponseHeaderCleanerHandler.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2ResponseHeaderCleanerHandler.java index ce3204954783..d8007771c930 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2ResponseHeaderCleanerHandler.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2ResponseHeaderCleanerHandler.java @@ -42,13 +42,13 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception } }); - super.channelRead(ctx, msg); + ctx.fireChannelRead(msg); } else if (msg instanceof Http2SettingsAckFrame) { ReferenceCountUtil.release(msg); } else if (msg instanceof Http2SettingsFrame) { Http2SettingsFrame settingsFrame = (Http2SettingsFrame)msg; logger.trace("SETTINGS retrieved - {}", settingsFrame.settings()); - super.channelRead(ctx, msg); + ctx.fireChannelRead(msg); } else { // Pass the message to the next handler in the pipeline ctx.fireChannelRead(msg); diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java index aec3eb243853..2c6dad20fc43 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java @@ -354,9 +354,11 @@ public Mono body() { bodyIntern().doOnDiscard( ByteBuf.class, buf -> { - if (leakDetectionDebuggingEnabled && buf.refCnt() > 0) { - buf.touch("ReactorNettyHttpResponse.body - onDiscard - refCnt: " + buf.refCnt()); - logger.info("ReactorNettyHttpResponse.body - onDiscard - refCnt: {}", buf.refCnt()); + if (buf.refCnt() > 0) { + if (leakDetectionDebuggingEnabled) { + buf.touch("ReactorNettyHttpResponse.body - onDiscard - refCnt: " + buf.refCnt()); + logger.info("ReactorNettyHttpResponse.body - onDiscard - refCnt: {}", buf.refCnt()); + } ReferenceCountUtil.safeRelease(buf); } }) @@ -418,9 +420,11 @@ private void releaseOnNotSubscribedResponse(ReactorNettyResponseState reactorNet body() .map(buf -> { - if (leakDetectionDebuggingEnabled && buf.refCnt() > 0) { - buf.touch("ReactorNettyHttpResponse.releaseOnNotSubscribedResponse - refCnt: " + buf.refCnt()); - logger.info("ReactorNettyHttpResponse.releaseOnNotSubscribedResponse - refCnt: {}", buf.refCnt()); + if (buf.refCnt() > 0) { + if (leakDetectionDebuggingEnabled) { + buf.touch("ReactorNettyHttpResponse.releaseOnNotSubscribedResponse - refCnt: " + buf.refCnt()); + logger.info("ReactorNettyHttpResponse.releaseOnNotSubscribedResponse - refCnt: {}", buf.refCnt()); + } ReferenceCountUtil.safeRelease(buf); } From 5b07ba2571eb2198f50f14df7f5f8bedbed98a86 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Sat, 15 Nov 2025 10:49:52 +0000 Subject: [PATCH 57/65] Update OrderbyDocumentQueryTest.java --- .../cosmos/rx/OrderbyDocumentQueryTest.java | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OrderbyDocumentQueryTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OrderbyDocumentQueryTest.java index b4df631aed8b..17c0938723a1 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OrderbyDocumentQueryTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OrderbyDocumentQueryTest.java @@ -10,6 +10,7 @@ import com.azure.cosmos.CosmosClientBuilder; import com.azure.cosmos.CosmosException; import com.azure.cosmos.CosmosItemSerializerNoExceptionWrapping; +import com.azure.cosmos.CosmosNettyLeakDetectorFactory; import com.azure.cosmos.implementation.AsyncDocumentClient; import com.azure.cosmos.implementation.FeedResponseDiagnostics; import com.azure.cosmos.implementation.FeedResponseListValidator; @@ -41,6 +42,7 @@ import io.reactivex.subscribers.TestSubscriber; import org.apache.commons.lang3.StringUtils; import org.testng.annotations.AfterClass; +import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; import org.testng.annotations.DataProvider; @@ -658,7 +660,12 @@ public List bulkInsert(CosmosAsyncContainer cosmosContainer, @BeforeMethod(groups = { "query" }) public void beforeMethod() throws Exception { // add a cool off time - TimeUnit.SECONDS.sleep(10); + CosmosNettyLeakDetectorFactory.resetIdentifiedLeaks(); + } + + @AfterMethod(groups = { "query" }, alwaysRun = true) + public void afterMethod() throws Exception { + logger.info("captureNettyLeaks: {}", captureNettyLeaks()); } @BeforeClass(groups = { "query" }, timeOut = 4 * SETUP_TIMEOUT) @@ -857,6 +864,30 @@ public void afterClass() { safeClose(client); } + public static String captureNettyLeaks() { + System.gc(); + try { + Thread.sleep(5_000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + List nettyLeaks = CosmosNettyLeakDetectorFactory.resetIdentifiedLeaks(); + if (nettyLeaks.size() > 0) { + StringBuilder sb = new StringBuilder(); + sb.append("\n"); + for (String leak : nettyLeaks) { + sb.append(leak).append("\n"); + } + + return "NETTY LEAKS detected: " + + "\n\n" + + sb; + } + + return ""; + } + + private void assertInvalidContinuationToken(String query, int[] pageSize, List expectedIds) { String requestContinuation = null; do { From d99560a2e60fc43abd3cebb7b62d22ee270c5bf6 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Mon, 17 Nov 2025 11:26:27 +0000 Subject: [PATCH 58/65] Test fixes --- .../azure/cosmos/ClientUnderTestBuilder.java | 6 ++ .../PerPartitionCircuitBreakerE2ETests.java | 17 ++++- ...aultInjectionConnectionErrorRuleTests.java | 7 +- .../com/azure/cosmos/rx/BackPressureTest.java | 1 + .../cosmos/rx/OrderbyDocumentQueryTest.java | 24 ------- .../com/azure/cosmos/rx/TestSuiteBase.java | 66 ++++++++++++++----- .../UserDefinedFunctionUpsertReplaceTest.java | 1 + .../IncrementalChangeFeedProcessorTest.java | 52 ++++----------- .../implementation/RxGatewayStoreModel.java | 18 ++--- 9 files changed, 99 insertions(+), 93 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ClientUnderTestBuilder.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ClientUnderTestBuilder.java index 9565d9cb630f..5c40da73f36e 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ClientUnderTestBuilder.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ClientUnderTestBuilder.java @@ -2,6 +2,7 @@ // Licensed under the MIT License. package com.azure.cosmos; +import com.azure.cosmos.implementation.AsyncDocumentClient; import com.azure.cosmos.implementation.ImplementationBridgeHelpers; import com.azure.cosmos.implementation.RxDocumentClientUnderTest; import com.azure.cosmos.implementation.Strings; @@ -59,6 +60,11 @@ public CosmosAsyncClient buildAsyncClient() { } catch (URISyntaxException e) { throw new IllegalArgumentException(e.getMessage()); } + + AsyncDocumentClient realAsyncClient = ReflectionUtils.getAsyncDocumentClient(cosmosAsyncClient); + if (realAsyncClient != null) { + realAsyncClient.close(); + } ReflectionUtils.setAsyncDocumentClient(cosmosAsyncClient, rxClient); return cosmosAsyncClient; } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/PerPartitionCircuitBreakerE2ETests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/PerPartitionCircuitBreakerE2ETests.java index 5503feab8382..10c5aa400894 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/PerPartitionCircuitBreakerE2ETests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/PerPartitionCircuitBreakerE2ETests.java @@ -54,7 +54,9 @@ import com.azure.cosmos.test.faultinjection.FaultInjectionServerErrorType; import org.testng.SkipException; import org.testng.annotations.AfterClass; +import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.DataProvider; import org.testng.annotations.Factory; import org.testng.annotations.Test; @@ -235,7 +237,7 @@ public PerPartitionCircuitBreakerE2ETests(CosmosClientBuilder cosmosClientBuilde super(cosmosClientBuilder); } - @BeforeClass(groups = {"circuit-breaker-misc-gateway", "circuit-breaker-misc-direct", "circuit-breaker-read-all-read-many"}) + @BeforeClass(groups = {"circuit-breaker-misc-gateway", "circuit-breaker-misc-direct", "circuit-breaker-read-all-read-many", "multi-region"}) public void beforeClass() { try (CosmosAsyncClient testClient = getClientBuilder().buildAsyncClient()) { RxDocumentClientImpl documentClient = (RxDocumentClientImpl) ReflectionUtils.getAsyncDocumentClient(testClient); @@ -4498,7 +4500,18 @@ private String resolveContainerIdByFaultInjectionOperationType(FaultInjectionOpe } } - @AfterClass(groups = {"circuit-breaker-misc-gateway", "circuit-breaker-misc-direct", "circuit-breaker-read-all-read-many"}) + @BeforeMethod(groups = { "circuit-breaker-misc-gateway", "circuit-breaker-misc-direct", "circuit-breaker-read-all-read-many", "multi-region" }, timeOut = 2 * SETUP_TIMEOUT, alwaysRun = true) + public void beforeMethod() throws Exception { + // add a cool off time + CosmosNettyLeakDetectorFactory.resetIdentifiedLeaks(); + } + + @AfterMethod(groups = { "circuit-breaker-misc-gateway", "circuit-breaker-misc-direct", "circuit-breaker-read-all-read-many", "multi-region" }, timeOut = SETUP_TIMEOUT, alwaysRun = true) + public void afterMethod() throws Exception { + logger.info("captureNettyLeaks: {}", captureNettyLeaks()); + } + + @AfterClass(groups = { "circuit-breaker-misc-gateway", "circuit-breaker-misc-direct", "circuit-breaker-read-all-read-many", "multi-region" }) public void afterClass() { CosmosClientBuilder clientBuilder = new CosmosClientBuilder() .endpoint(TestConfigurations.HOST) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/FaultInjectionConnectionErrorRuleTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/FaultInjectionConnectionErrorRuleTests.java index b3eedd79ff17..d3301b2de255 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/FaultInjectionConnectionErrorRuleTests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/FaultInjectionConnectionErrorRuleTests.java @@ -97,6 +97,7 @@ public void faultInjectionConnectionErrorRuleTestWithNoConnectionWarmup(FaultInj .contentResponseOnWriteEnabled(true) .directMode() .buildAsyncClient(); + FaultInjectionRule connectionErrorRule = null; try { // using single partition here so that all write operations will be on the same physical partitions @@ -113,7 +114,7 @@ public void faultInjectionConnectionErrorRuleTestWithNoConnectionWarmup(FaultInj // now enable the connection error rule which expected to close the connections String ruleId = "connectionErrorRule-close-" + UUID.randomUUID(); - FaultInjectionRule connectionErrorRule = + connectionErrorRule = new FaultInjectionRuleBuilder(ruleId) .condition( new FaultInjectionConditionBuilder() @@ -158,6 +159,10 @@ public void faultInjectionConnectionErrorRuleTestWithNoConnectionWarmup(FaultInj } catch (InterruptedException ex) { throw new RuntimeException(ex); } finally { + if (connectionErrorRule != null) { + connectionErrorRule.disable(); + } + safeClose(client); } } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/BackPressureTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/BackPressureTest.java index f44802a44e4a..1a3c89736e8c 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/BackPressureTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/BackPressureTest.java @@ -251,6 +251,7 @@ public void queryItems() throws Exception { public void before_BackPressureTest() throws Exception { CosmosContainerRequestOptions options = new CosmosContainerRequestOptions(); + client = new ClientUnderTestBuilder( getClientBuilder() .key(TestConfigurations.MASTER_KEY) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OrderbyDocumentQueryTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OrderbyDocumentQueryTest.java index 17c0938723a1..e210696bad82 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OrderbyDocumentQueryTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/OrderbyDocumentQueryTest.java @@ -864,30 +864,6 @@ public void afterClass() { safeClose(client); } - public static String captureNettyLeaks() { - System.gc(); - try { - Thread.sleep(5_000); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - List nettyLeaks = CosmosNettyLeakDetectorFactory.resetIdentifiedLeaks(); - if (nettyLeaks.size() > 0) { - StringBuilder sb = new StringBuilder(); - sb.append("\n"); - for (String leak : nettyLeaks) { - sb.append(leak).append("\n"); - } - - return "NETTY LEAKS detected: " - + "\n\n" - + sb; - } - - return ""; - } - - private void assertInvalidContinuationToken(String query, int[] pageSize, List expectedIds) { String requestContinuation = null; do { diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java index e17c95012704..18be615dc220 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java @@ -280,27 +280,35 @@ protected static void cleanUpContainer(CosmosAsyncContainer cosmosContainer) { } protected static void truncateCollection(CosmosAsyncContainer cosmosContainer) { - int i = 0; - while (i < 100) { - try { - truncateCollectionInternal(cosmosContainer); - return; - } catch (CosmosException exception) { - if (exception.getStatusCode() != HttpConstants.StatusCodes.TOO_MANY_REQUESTS - || exception.getSubStatusCode() != 3200) { - - logger.error("No retry of exception", exception); - throw exception; - } - i++; - logger.info("Retrying truncation after 100ms - iteration " + i); + try { + int i = 0; + while (i < 100) { try { - Thread.sleep(100); - } catch (InterruptedException e) { - throw new RuntimeException(e); + truncateCollectionInternal(cosmosContainer); + return; + } catch (CosmosException exception) { + if (exception.getStatusCode() != HttpConstants.StatusCodes.TOO_MANY_REQUESTS + || exception.getSubStatusCode() != 3200) { + + logger.error("No retry of exception", exception); + throw exception; + } + + i++; + logger.info("Retrying truncation after 100ms - iteration " + i); + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } } } + } finally { + // TODO @fabianm - Resetting leak detection - there is some flakiness when tests + // truncate collection and hit throttling - this will need to be investigated + // separately + CosmosNettyLeakDetectorFactory.resetIdentifiedLeaks(); } } @@ -1567,4 +1575,28 @@ public byte[] decodeHexString(String string) { } return outputStream.toByteArray(); } + + public static String captureNettyLeaks() { + System.gc(); + try { + Thread.sleep(5_000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + List nettyLeaks = CosmosNettyLeakDetectorFactory.resetIdentifiedLeaks(); + if (nettyLeaks.size() > 0) { + StringBuilder sb = new StringBuilder(); + sb.append("\n"); + for (String leak : nettyLeaks) { + sb.append(leak).append("\n"); + } + + return "NETTY LEAKS detected: " + + "\n\n" + + sb; + } + + return ""; + } + } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/UserDefinedFunctionUpsertReplaceTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/UserDefinedFunctionUpsertReplaceTest.java index 5d8dff9eacd3..c9d19969c8dc 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/UserDefinedFunctionUpsertReplaceTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/UserDefinedFunctionUpsertReplaceTest.java @@ -4,6 +4,7 @@ import com.azure.cosmos.CosmosAsyncClient; import com.azure.cosmos.CosmosAsyncContainer; +import com.azure.cosmos.CosmosNettyLeakDetectorFactory; import com.azure.cosmos.models.CosmosUserDefinedFunctionResponse; import com.azure.cosmos.CosmosClientBuilder; import com.azure.cosmos.CosmosResponseValidator; diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/changefeed/pkversion/IncrementalChangeFeedProcessorTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/changefeed/pkversion/IncrementalChangeFeedProcessorTest.java index a8300df95f85..ad70d2997423 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/changefeed/pkversion/IncrementalChangeFeedProcessorTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/changefeed/pkversion/IncrementalChangeFeedProcessorTest.java @@ -10,6 +10,7 @@ import com.azure.cosmos.CosmosAsyncDatabase; import com.azure.cosmos.CosmosClientBuilder; import com.azure.cosmos.CosmosEndToEndOperationLatencyPolicyConfigBuilder; +import com.azure.cosmos.CosmosNettyLeakDetectorFactory; import com.azure.cosmos.SplitTestsRetryAnalyzer; import com.azure.cosmos.SplitTimeoutException; import com.azure.cosmos.ThroughputControlGroupConfig; @@ -2207,53 +2208,24 @@ private Consumer> leasesChangeFeedProcessorHandler(LeaseStateMoni log.info("LEASES processing from thread {}", Thread.currentThread().getId()); }; } + @BeforeMethod(groups = { "long-emulator", "cfp-split", "multi-master", "query" }, timeOut = 2 * SETUP_TIMEOUT, alwaysRun = true) + public void beforeMethod() throws Exception { + // add a cool off time + CosmosNettyLeakDetectorFactory.resetIdentifiedLeaks(); + } - @BeforeMethod(groups = { "long-emulator", "cfp-split" }, timeOut = 2 * SETUP_TIMEOUT, alwaysRun = true) - public void beforeMethod() { - } + @AfterMethod(groups = { "long-emulator", "cfp-split", "multi-master", "query" }, timeOut = SETUP_TIMEOUT, alwaysRun = true) + public void afterMethod() throws Exception { + logger.info("captureNettyLeaks: {}", captureNettyLeaks()); + } - @BeforeClass(groups = { "long-emulator", "cfp-split" }, timeOut = SETUP_TIMEOUT, alwaysRun = true) + @BeforeClass(groups = { "long-emulator", "cfp-split", "multi-master", "query" }, timeOut = SETUP_TIMEOUT, alwaysRun = true) public void before_ChangeFeedProcessorTest() { client = getClientBuilder().buildAsyncClient(); createdDatabase = getSharedCosmosDatabase(client); - - // Following is code that when enabled locally it allows for a predicted database/collection name that can be - // checked in the Azure Portal -// try { -// client.getDatabase(databaseId).read() -// .map(cosmosDatabaseResponse -> cosmosDatabaseResponse.getDatabase()) -// .flatMap(database -> database.delete()) -// .onErrorResume(throwable -> { -// if (throwable instanceof com.azure.cosmos.CosmosClientException) { -// com.azure.cosmos.CosmosClientException clientException = (com.azure.cosmos.CosmosClientException) throwable; -// if (clientException.getStatusCode() == 404) { -// return Mono.empty(); -// } -// } -// return Mono.error(throwable); -// }).block(); -// Thread.sleep(500); -// } catch (Exception e){ -// log.warn("Database delete", e); -// } -// createdDatabase = createDatabase(client, databaseId); } - - @AfterMethod(groups = { "long-emulator", "cfp-split" }, timeOut = 3 * SHUTDOWN_TIMEOUT, alwaysRun = true) - public void afterMethod() { - } - - @AfterClass(groups = { "long-emulator", "cfp-split" }, timeOut = 2 * SHUTDOWN_TIMEOUT, alwaysRun = true) + @AfterClass(groups = { "long-emulator", "cfp-split", "multi-master", "query" }, timeOut = 2 * SHUTDOWN_TIMEOUT, alwaysRun = true) public void afterClass() { -// try { -// client.readAllDatabases() -// .flatMap(cosmosDatabaseProperties -> { -// CosmosAsyncDatabase cosmosDatabase = client.getDatabase(cosmosDatabaseProperties.getId()); -// return cosmosDatabase.delete(); -// }).blockLast(); -// Thread.sleep(500); -// } catch (Exception e){ } - safeClose(client); } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java index cdb4c29cb486..8113d261d04d 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxGatewayStoreModel.java @@ -214,7 +214,7 @@ public StoreResponse unwrapToStoreResponse( if (leakDetectionDebuggingEnabled) { retainedContent.touch( "RxGatewayStoreModel.unwrapToStoreResponse before validate - refCnt: " + retainedContent.refCnt()); - logger.info("RxGatewayStoreModel.unwrapToStoreResponse before validate - refCnt: {}", retainedContent.refCnt()); + logger.debug("RxGatewayStoreModel.unwrapToStoreResponse before validate - refCnt: {}", retainedContent.refCnt()); } // If there is any error in the header response this throws exception @@ -224,7 +224,7 @@ public StoreResponse unwrapToStoreResponse( if ((size = retainedContent.readableBytes()) > 0) { if (leakDetectionDebuggingEnabled) { retainedContent.touch("RxGatewayStoreModel before creating StoreResponse - refCnt: " + retainedContent.refCnt()); - logger.info("RxGatewayStoreModel before creating StoreResponse - refCnt: {}", retainedContent.refCnt()); + logger.debug("RxGatewayStoreModel before creating StoreResponse - refCnt: {}", retainedContent.refCnt()); } return new StoreResponse( @@ -427,7 +427,7 @@ private Mono toDocumentServiceResponse(Mono { if (leakDetectionDebuggingEnabled) { bodyByteBuf.touch("RxGatewayStoreModel - buffer after aggregate before retain - refCnt: " + bodyByteBuf.refCnt()); - logger.info("RxGatewayStoreModel - buffer after aggregate before retain - refCnt: {}", bodyByteBuf.refCnt()); + logger.debug("RxGatewayStoreModel - buffer after aggregate before retain - refCnt: {}", bodyByteBuf.refCnt()); } if (bodyByteBuf != Unpooled.EMPTY_BUFFER) { @@ -437,7 +437,7 @@ private Mono toDocumentServiceResponse(Mono toDocumentServiceResponse(Mono 0) { if (leakDetectionDebuggingEnabled) { buf.touch("RxGatewayStoreModel - doOnDiscard - begin - refCnt: " + buf.refCnt()); - logger.info("RxGatewayStoreModel - doOnDiscard - begin - refCnt: {}", buf.refCnt()); + logger.debug("RxGatewayStoreModel - doOnDiscard - begin - refCnt: {}", buf.refCnt()); } // there could be a race with the catch in the .map operator below @@ -460,7 +460,7 @@ private Mono toDocumentServiceResponse(Mono { if (leakDetectionDebuggingEnabled) { content.touch("RxGatewayStoreModel - before capturing transport timeline - refCnt: " + content.refCnt()); - logger.info("RxGatewayStoreModel - before capturing transport timeline - refCnt: {}", content.refCnt()); + logger.debug("RxGatewayStoreModel - before capturing transport timeline - refCnt: {}", content.refCnt()); } try { @@ -472,7 +472,7 @@ private Mono toDocumentServiceResponse(Mono toDocumentServiceResponse(Mono 0) { if (leakDetectionDebuggingEnabled) { content.touch("RxGatewayStoreModel -exception creating StoreResponse - refCnt: " + content.refCnt()); - logger.info("RxGatewayStoreModel -exception creating StoreResponse - refCnt: {}", content.refCnt()); + logger.debug("RxGatewayStoreModel -exception creating StoreResponse - refCnt: {}", content.refCnt()); } // Unwrap failed before StoreResponse took ownership -> release our retain // there could be a race with the doOnDiscard above - so, use safeRelease @@ -520,7 +520,7 @@ private Mono toDocumentServiceResponse(Mono 0) { if (leakDetectionDebuggingEnabled) { buf.touch("RxGatewayStoreModel - doOnDiscard after map - refCnt: " + buf.refCnt()); - logger.info("RxGatewayStoreModel - doOnDiscard after map - refCnt: {}", buf.refCnt()); + logger.debug("RxGatewayStoreModel - doOnDiscard after map - refCnt: {}", buf.refCnt()); } ReferenceCountUtil.safeRelease(buf); } From f73d4e056d4f382139f81ffb4a180f70a97f9370 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Mon, 17 Nov 2025 23:30:58 +0000 Subject: [PATCH 59/65] Reverting too noisy logs --- .../implementation/AsyncDocumentClient.java | 8 +------- .../rntbd/RntbdContextDecoder.java | 6 +++--- .../rntbd/RntbdResponseDecoder.java | 4 ++-- .../http/Http2ResponseHeaderCleanerHandler.java | 4 ++-- .../implementation/http/ReactorNettyClient.java | 8 ++------ sdk/cosmos/live-platform-matrix.json | 16 ++++++++-------- 6 files changed, 18 insertions(+), 28 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/AsyncDocumentClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/AsyncDocumentClient.java index 9baeb17dbde8..fd2f002cf9ba 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/AsyncDocumentClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/AsyncDocumentClient.java @@ -329,13 +329,7 @@ public AsyncDocumentClient build() { operationPolicies, isPerPartitionAutomaticFailoverEnabled); - try { - client.init(state, null); - } catch (Throwable t) { - client.close(); - // TODO - should we map this to a CosmosException? - throw t; - } + client.init(state, null); return client; } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdContextDecoder.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdContextDecoder.java index 13e953e47734..353f34461aa1 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdContextDecoder.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdContextDecoder.java @@ -44,14 +44,14 @@ protected void decode(final ChannelHandlerContext context, final ByteBuf in, fin result = rntbdContext; if (leakDetectionDebuggingEnabled) { - logger.info("{} RntbdContextDecoder: decoded RntbdContext successfully", context.channel()); + logger.debug("{} RntbdContextDecoder: decoded RntbdContext successfully", context.channel()); } } catch (RntbdContextException error) { context.fireUserEventTriggered(error); result = error; if (leakDetectionDebuggingEnabled) { - logger.info("{} RntbdContextDecoder: caught RntbdContextException", context.channel(), error); + logger.debug("{} RntbdContextDecoder: caught RntbdContextException", context.channel(), error); } } finally { if (leakDetectionDebuggingEnabled) { @@ -62,7 +62,7 @@ protected void decode(final ChannelHandlerContext context, final ByteBuf in, fin logger.debug("{} DECODE COMPLETE: {}", context.channel(), result); } else if (leakDetectionDebuggingEnabled) { - logger.info("{} RntbdContextDecoder: cannot decode head yet, readableBytes={}", + logger.debug("{} RntbdContextDecoder: cannot decode head yet, readableBytes={}", context.channel(), in.readableBytes()); } } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdResponseDecoder.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdResponseDecoder.java index 8ae5da0aa315..a946dfe8bce2 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdResponseDecoder.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdResponseDecoder.java @@ -65,11 +65,11 @@ protected void decode(final ChannelHandlerContext context, final ByteBuf in, fin out.add(response.retain()); } else if (leakDetectionDebuggingEnabled) { - logger.info("{} RntbdResponseDecoder: response is null, not enough data to decode yet", + logger.debug("{} RntbdResponseDecoder: response is null, not enough data to decode yet", context.channel()); } } else if (leakDetectionDebuggingEnabled) { - logger.info("{} RntbdResponseDecoder: cannot decode head yet, readableBytes={}", + logger.debug("{} RntbdResponseDecoder: cannot decode head yet, readableBytes={}", context.channel(), in.readableBytes()); } } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2ResponseHeaderCleanerHandler.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2ResponseHeaderCleanerHandler.java index d8007771c930..ce3204954783 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2ResponseHeaderCleanerHandler.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2ResponseHeaderCleanerHandler.java @@ -42,13 +42,13 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception } }); - ctx.fireChannelRead(msg); + super.channelRead(ctx, msg); } else if (msg instanceof Http2SettingsAckFrame) { ReferenceCountUtil.release(msg); } else if (msg instanceof Http2SettingsFrame) { Http2SettingsFrame settingsFrame = (Http2SettingsFrame)msg; logger.trace("SETTINGS retrieved - {}", settingsFrame.settings()); - ctx.fireChannelRead(msg); + super.channelRead(ctx, msg); } else { // Pass the message to the next handler in the pipeline ctx.fireChannelRead(msg); diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java index 2c6dad20fc43..44e2d0045edb 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java @@ -357,7 +357,7 @@ public Mono body() { if (buf.refCnt() > 0) { if (leakDetectionDebuggingEnabled) { buf.touch("ReactorNettyHttpResponse.body - onDiscard - refCnt: " + buf.refCnt()); - logger.info("ReactorNettyHttpResponse.body - onDiscard - refCnt: {}", buf.refCnt()); + logger.debug("ReactorNettyHttpResponse.body - onDiscard - refCnt: {}", buf.refCnt()); } ReferenceCountUtil.safeRelease(buf); } @@ -414,16 +414,12 @@ private void releaseOnNotSubscribedResponse(ReactorNettyResponseState reactorNet logger.debug("Releasing body, not yet subscribed"); } - if (leakDetectionDebuggingEnabled) { - logger.info("Releasing body, not yet subscribed"); - } - body() .map(buf -> { if (buf.refCnt() > 0) { if (leakDetectionDebuggingEnabled) { buf.touch("ReactorNettyHttpResponse.releaseOnNotSubscribedResponse - refCnt: " + buf.refCnt()); - logger.info("ReactorNettyHttpResponse.releaseOnNotSubscribedResponse - refCnt: {}", buf.refCnt()); + logger.debug("ReactorNettyHttpResponse.releaseOnNotSubscribedResponse - refCnt: {}", buf.refCnt()); } ReferenceCountUtil.safeRelease(buf); } diff --git a/sdk/cosmos/live-platform-matrix.json b/sdk/cosmos/live-platform-matrix.json index 613390ddd671..7f3e3af80997 100644 --- a/sdk/cosmos/live-platform-matrix.json +++ b/sdk/cosmos/live-platform-matrix.json @@ -14,7 +14,7 @@ "-Pcircuit-breaker-read-all-read-many": "CircuitBreakerReadAllAndReadMany", "-Pmulti-region": "MultiRegion", "-Plong": "Long", - "-DargLine=\"-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -Dazure.cosmos.directModeProtocol=Tcp\"": "TCP", + "-DargLine=\"-Dazure.cosmos.directModeProtocol=Tcp\"": "TCP", "Session": "", "ubuntu": "", "@{ enableMultipleWriteLocations = $false; defaultConsistencyLevel = 'Session' }": "", @@ -38,7 +38,7 @@ } }, "AdditionalArgs": [ - "-DargLine=\"-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -Dazure.cosmos.directModeProtocol=Tcp\"" + "-DargLine=\"-Dazure.cosmos.directModeProtocol=Tcp\"" ], "ProfileFlag": "-Pe2e", "Agent": { @@ -52,7 +52,7 @@ "ProfileFlag": [ "-Pcfp-split", "-Psplit", "-Pquery", "-Pfast", "-Pdirect" ], "ArmTemplateParameters": "@{ enableMultipleWriteLocations = $false; defaultConsistencyLevel = 'Session' }", "AdditionalArgs": [ - "-DargLine=\"-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -Dio.netty.handler.ssl.noOpenSsl=true\"" + "-DargLine=\"-Dio.netty.handler.ssl.noOpenSsl=true\"" ], "Agent": { "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" } @@ -71,7 +71,7 @@ { "DESIRED_CONSISTENCY": "BoundedStaleness", "ACCOUNT_CONSISTENCY": "Strong", - "AdditionalArgs": "-DargLine=\"-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -Dazure.cosmos.directModeProtocol=Tcp\"", + "AdditionalArgs": "-DargLine=\"-Dazure.cosmos.directModeProtocol=Tcp\"", "ProfileFlag": "-Pe2e", "ArmTemplateParameters": "@{ enableMultipleWriteLocations = $false; defaultConsistencyLevel = 'Strong' }", "Agent": { @@ -109,7 +109,7 @@ }, "PROTOCOLS": "[\"Tcp\"]", "ProfileFlag": [ "-Pmulti-master" ], - "AdditionalArgs": "\"-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -DCOSMOS.PARTITION_LEVEL_CIRCUIT_BREAKER_DEFAULT_CONFIG_OPT_IN=TRUE\"", + "AdditionalArgs": "\"-DCOSMOS.PARTITION_LEVEL_CIRCUIT_BREAKER_DEFAULT_CONFIG_OPT_IN=TRUE\"", "Agent": { "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" } } @@ -125,7 +125,7 @@ }, "PROTOCOLS": "[\"Tcp\"]", "ProfileFlag": [ "-Pfi-multi-master" ], - "AdditionalArgs": "\"-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -DCOSMOS.PARTITION_LEVEL_CIRCUIT_BREAKER_DEFAULT_CONFIG_OPT_IN=TRUE\"", + "AdditionalArgs": "\"-DCOSMOS.PARTITION_LEVEL_CIRCUIT_BREAKER_DEFAULT_CONFIG_OPT_IN=TRUE\"", "Agent": { "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" } } @@ -141,7 +141,7 @@ }, "PROTOCOLS": "[\"Tcp\"]", "ProfileFlag": [ "-Pmulti-master" ], - "AdditionalArgs": "\"-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -DCOSMOS.PARTITION_LEVEL_CIRCUIT_BREAKER_DEFAULT_CONFIG_OPT_IN=FALSE\"", + "AdditionalArgs": "\"-DCOSMOS.PARTITION_LEVEL_CIRCUIT_BREAKER_DEFAULT_CONFIG_OPT_IN=FALSE\"", "Agent": { "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" } } @@ -157,7 +157,7 @@ }, "PROTOCOLS": "[\"Tcp\"]", "ProfileFlag": [ "-Pfi-multi-master" ], - "AdditionalArgs": "\"-DCOSMOS.CLIENT_LEAK_DETECTION_ENABLED=true -DCOSMOS.PARTITION_LEVEL_CIRCUIT_BREAKER_DEFAULT_CONFIG_OPT_IN=FALSE\"", + "AdditionalArgs": "\"-DCOSMOS.PARTITION_LEVEL_CIRCUIT_BREAKER_DEFAULT_CONFIG_OPT_IN=FALSE\"", "Agent": { "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" } } From 65950267a1b6c1055d1d2f2c186de07dc1d5f327 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Tue, 18 Nov 2025 18:44:05 +0000 Subject: [PATCH 60/65] Test fixes --- sdk/cosmos/azure-cosmos-benchmark/pom.xml | 1 - .../cosmos/rx/MultiOrderByQueryTests.java | 11 +++++++---- .../com/azure/cosmos/rx/TestSuiteBase.java | 18 ++++++++++++++++++ 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-benchmark/pom.xml b/sdk/cosmos/azure-cosmos-benchmark/pom.xml index 5c9138a92de4..d0691c7a71b8 100644 --- a/sdk/cosmos/azure-cosmos-benchmark/pom.xml +++ b/sdk/cosmos/azure-cosmos-benchmark/pom.xml @@ -206,7 +206,6 @@ Licensed under the MIT License. true 1 - 256 paranoid diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/MultiOrderByQueryTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/MultiOrderByQueryTests.java index c6e232ec7b84..a5e9e832aa1c 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/MultiOrderByQueryTests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/MultiOrderByQueryTests.java @@ -16,8 +16,8 @@ import com.azure.cosmos.util.CosmosPagedFlux; import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.collections4.ComparatorUtils; -import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeClass; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.Factory; import org.testng.annotations.Test; @@ -110,17 +110,19 @@ public MultiOrderByQueryTests(CosmosClientBuilder clientBuilder) { super(clientBuilder); } - @AfterClass(groups = { "query" }, timeOut = SHUTDOWN_TIMEOUT, alwaysRun = true) + @AfterMethod(groups = { "query" }, timeOut = SHUTDOWN_TIMEOUT, alwaysRun = true) public void afterClass() { safeClose(client); } - @BeforeClass(groups = { "query" }, timeOut = SETUP_TIMEOUT) + @BeforeMethod(groups = { "query" }, timeOut = SETUP_TIMEOUT) public void before_MultiOrderByQueryTests() throws Exception { + documents = new ArrayList<>(); client = getClientBuilder().buildAsyncClient(); documentCollection = getSharedMultiPartitionCosmosContainerWithCompositeAndSpatialIndexes(client); truncateCollection(documentCollection); + expectCount(documentCollection, 0); int numberOfDocuments = 4; Random random = new Random(); @@ -160,6 +162,7 @@ public void before_MultiOrderByQueryTests() throws Exception { } voidBulkInsertBlocking(documentCollection, documents); + expectCount(documentCollection, documents.size()); waitIfNeededForReplicasToCatchUp(getClientBuilder()); } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java index 18be615dc220..64b1103fa46c 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java @@ -312,6 +312,21 @@ protected static void truncateCollection(CosmosAsyncContainer cosmosContainer) { } } + protected static void expectCount(CosmosAsyncContainer cosmosContainer, int expectedCount) { + CosmosQueryRequestOptions options = new CosmosQueryRequestOptions(); + options.setCosmosEndToEndOperationLatencyPolicyConfig( + new CosmosEndToEndOperationLatencyPolicyConfigBuilder(Duration.ofHours(1)) + .build() + ); + options.setMaxDegreeOfParallelism(-1); + List counts = cosmosContainer + .queryItems("SELECT VALUE COUNT(0) FROM root", options, Integer.class) + .collectList() + .block(); + assertThat(counts).hasSize(1); + assertThat(counts.get(0)).isEqualTo(expectedCount); + } + private static void truncateCollectionInternal(CosmosAsyncContainer cosmosContainer) { CosmosContainerProperties cosmosContainerProperties = cosmosContainer.read().block().getProperties(); String cosmosContainerId = cosmosContainerProperties.getId(); @@ -350,6 +365,9 @@ private static void truncateCollectionInternal(CosmosAsyncContainer cosmosContai return cosmosContainer.deleteItem(doc.getId(), partitionKey); }).then().block(); + + expectCount(cosmosContainer, 0); + logger.info("Truncating collection {} triggers ...", cosmosContainerId); cosmosContainer.getScripts().queryTriggers("SELECT * FROM root", options) From 92fb0545b9a2f396b3b870fad735824e11279e96 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Tue, 18 Nov 2025 18:54:30 +0000 Subject: [PATCH 61/65] Fixing POMs --- sdk/cosmos/azure-cosmos-benchmark/pom.xml | 27 +++++++------ sdk/cosmos/azure-cosmos-encryption/pom.xml | 6 +-- sdk/cosmos/azure-cosmos-kafka-connect/pom.xml | 10 ++--- .../pom.xml | 2 +- .../azure-cosmos-spark_3-3_2-12/pom.xml | 2 +- .../azure-cosmos-spark_3-4_2-12/pom.xml | 2 +- .../azure-cosmos-spark_3-5_2-12/pom.xml | 2 +- sdk/cosmos/azure-cosmos-spark_3/pom.xml | 2 +- sdk/cosmos/azure-cosmos-test/pom.xml | 2 +- sdk/cosmos/azure-cosmos-tests/pom.xml | 40 +++++++++---------- sdk/cosmos/azure-cosmos/pom.xml | 2 +- sdk/cosmos/fabric-cosmos-spark-auth_3/pom.xml | 2 +- 12 files changed, 50 insertions(+), 49 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-benchmark/pom.xml b/sdk/cosmos/azure-cosmos-benchmark/pom.xml index d0691c7a71b8..d865417c6dca 100644 --- a/sdk/cosmos/azure-cosmos-benchmark/pom.xml +++ b/sdk/cosmos/azure-cosmos-benchmark/pom.xml @@ -206,6 +206,7 @@ Licensed under the MIT License. true 1 + 256 paranoid @@ -320,7 +321,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid @@ -347,7 +348,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid @@ -374,7 +375,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid @@ -401,7 +402,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid @@ -428,7 +429,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid @@ -455,7 +456,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid @@ -482,7 +483,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid @@ -509,7 +510,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid @@ -536,7 +537,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid @@ -563,7 +564,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid @@ -591,7 +592,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid @@ -626,7 +627,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid @@ -653,7 +654,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid diff --git a/sdk/cosmos/azure-cosmos-encryption/pom.xml b/sdk/cosmos/azure-cosmos-encryption/pom.xml index e4810e35139d..1a2eeb22d564 100644 --- a/sdk/cosmos/azure-cosmos-encryption/pom.xml +++ b/sdk/cosmos/azure-cosmos-encryption/pom.xml @@ -222,7 +222,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid @@ -278,7 +278,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid @@ -305,7 +305,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml b/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml index f3663f541eef..91a2f09aea36 100644 --- a/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml +++ b/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml @@ -279,7 +279,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid @@ -546,7 +546,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid @@ -573,7 +573,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid @@ -600,7 +600,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid @@ -627,7 +627,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid diff --git a/sdk/cosmos/azure-cosmos-spark-account-data-resolver-sample/pom.xml b/sdk/cosmos/azure-cosmos-spark-account-data-resolver-sample/pom.xml index a9c332d7af1c..df475d835cfb 100644 --- a/sdk/cosmos/azure-cosmos-spark-account-data-resolver-sample/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark-account-data-resolver-sample/pom.xml @@ -699,7 +699,7 @@ true 1 - 256 + 256 paranoid diff --git a/sdk/cosmos/azure-cosmos-spark_3-3_2-12/pom.xml b/sdk/cosmos/azure-cosmos-spark_3-3_2-12/pom.xml index 7b797b4cd36f..315bc2259f92 100644 --- a/sdk/cosmos/azure-cosmos-spark_3-3_2-12/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark_3-3_2-12/pom.xml @@ -121,7 +121,7 @@ true 1 - 256 + 256 paranoid diff --git a/sdk/cosmos/azure-cosmos-spark_3-4_2-12/pom.xml b/sdk/cosmos/azure-cosmos-spark_3-4_2-12/pom.xml index afaa80a9e50f..0ec4dd9d214a 100644 --- a/sdk/cosmos/azure-cosmos-spark_3-4_2-12/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark_3-4_2-12/pom.xml @@ -121,7 +121,7 @@ true 1 - 256 + 256 paranoid diff --git a/sdk/cosmos/azure-cosmos-spark_3-5_2-12/pom.xml b/sdk/cosmos/azure-cosmos-spark_3-5_2-12/pom.xml index a8ffcf3b2d5d..11f0ab4f685a 100644 --- a/sdk/cosmos/azure-cosmos-spark_3-5_2-12/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark_3-5_2-12/pom.xml @@ -121,7 +121,7 @@ true 1 - 256 + 256 paranoid diff --git a/sdk/cosmos/azure-cosmos-spark_3/pom.xml b/sdk/cosmos/azure-cosmos-spark_3/pom.xml index 58e12cd940f0..11ca06106078 100644 --- a/sdk/cosmos/azure-cosmos-spark_3/pom.xml +++ b/sdk/cosmos/azure-cosmos-spark_3/pom.xml @@ -750,7 +750,7 @@ true 1 - 256 + 256 paranoid diff --git a/sdk/cosmos/azure-cosmos-test/pom.xml b/sdk/cosmos/azure-cosmos-test/pom.xml index fe5dee23f566..7ef7ada263d7 100644 --- a/sdk/cosmos/azure-cosmos-test/pom.xml +++ b/sdk/cosmos/azure-cosmos-test/pom.xml @@ -155,7 +155,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid diff --git a/sdk/cosmos/azure-cosmos-tests/pom.xml b/sdk/cosmos/azure-cosmos-tests/pom.xml index e5898d7c8f8b..5093ced080dd 100644 --- a/sdk/cosmos/azure-cosmos-tests/pom.xml +++ b/sdk/cosmos/azure-cosmos-tests/pom.xml @@ -236,7 +236,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid @@ -327,7 +327,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid @@ -354,7 +354,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid @@ -381,7 +381,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid @@ -408,7 +408,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid @@ -435,7 +435,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid @@ -462,7 +462,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid @@ -489,7 +489,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid @@ -516,7 +516,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid @@ -543,7 +543,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid @@ -570,7 +570,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid @@ -597,7 +597,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid @@ -624,7 +624,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid @@ -651,7 +651,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid @@ -678,7 +678,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid @@ -706,7 +706,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid @@ -741,7 +741,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid @@ -768,7 +768,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid @@ -795,7 +795,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid @@ -822,7 +822,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid diff --git a/sdk/cosmos/azure-cosmos/pom.xml b/sdk/cosmos/azure-cosmos/pom.xml index c11f2a13b1e0..ef7d198e3dde 100644 --- a/sdk/cosmos/azure-cosmos/pom.xml +++ b/sdk/cosmos/azure-cosmos/pom.xml @@ -149,7 +149,7 @@ Licensed under the MIT License. true 1 - 256 + 256 paranoid diff --git a/sdk/cosmos/fabric-cosmos-spark-auth_3/pom.xml b/sdk/cosmos/fabric-cosmos-spark-auth_3/pom.xml index 75035c197bc1..bfd53130788c 100644 --- a/sdk/cosmos/fabric-cosmos-spark-auth_3/pom.xml +++ b/sdk/cosmos/fabric-cosmos-spark-auth_3/pom.xml @@ -509,7 +509,7 @@ true 1 - 256 + 256 paranoid From c4d216dea6c846a698bdb9cec168f429fcc3d835 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Tue, 18 Nov 2025 18:58:56 +0000 Subject: [PATCH 62/65] React to code review feedback --- .../cosmos/encryption/CosmosNettyLeakDetectorFactory.java | 6 ------ .../com/azure/cosmos/CosmosNettyLeakDetectorFactory.java | 6 ------ 2 files changed, 12 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java index 759b4089e6c8..7fd1b398da22 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java @@ -206,12 +206,6 @@ public static void ingestIntoNetty() { return; } - // sample every allocation - System.setProperty("io.netty.leakDetection.samplingInterval", "1"); - System.setProperty("io.netty.leakDetection.targetRecords", "256"); - // Must run before any Netty ByteBuf is allocated - ResourceLeakDetector.setLevel(ResourceLeakDetector.Level.PARANOID); - // install custom reporter ResourceLeakDetectorFactory.setResourceLeakDetectorFactory(new CosmosNettyLeakDetectorFactory()); diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java index 58d80e48e54e..c4a2ee9b6710 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java @@ -206,12 +206,6 @@ public static void ingestIntoNetty() { return; } - // sample every allocation - System.setProperty("io.netty.leakDetection.samplingInterval", "1"); - System.setProperty("io.netty.leakDetection.targetRecords", "256"); - // Must run before any Netty ByteBuf is allocated - ResourceLeakDetector.setLevel(ResourceLeakDetector.Level.PARANOID); - // install custom reporter ResourceLeakDetectorFactory.setResourceLeakDetectorFactory(new CosmosNettyLeakDetectorFactory()); From e9d893e5dd82f1ab842d7f8f92ee5a3384e64a72 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Tue, 18 Nov 2025 19:01:22 +0000 Subject: [PATCH 63/65] Update pom.xml --- sdk/cosmos/azure-cosmos-tests/pom.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sdk/cosmos/azure-cosmos-tests/pom.xml b/sdk/cosmos/azure-cosmos-tests/pom.xml index 5093ced080dd..3c343e2d459f 100644 --- a/sdk/cosmos/azure-cosmos-tests/pom.xml +++ b/sdk/cosmos/azure-cosmos-tests/pom.xml @@ -848,6 +848,9 @@ Licensed under the MIT License. true + 1 + 256 + paranoid From 4c6cc4cef01a9230a24386a4eb4a015e8758f8a2 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Tue, 18 Nov 2025 19:05:06 +0000 Subject: [PATCH 64/65] Addresses code review feedback --- .../cosmos/encryption/CosmosNettyLeakDetectorFactory.java | 3 ++- .../java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java index 7fd1b398da22..83a975352e0b 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/encryption/CosmosNettyLeakDetectorFactory.java @@ -118,7 +118,8 @@ private void onAfterClassCore(ITestClass testClass) { synchronized (staticLock) { instanceCountSnapshot = testClassInventory.get(testClassName); if (instanceCountSnapshot == null) { - testClassInventory.put(testClassName, instanceCountSnapshot = new AtomicInteger(0)); + throw new IllegalStateException( + "BeforeClass in TestListener was not called for testClass " + testClassName); } } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java index c4a2ee9b6710..c8c87f945a29 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNettyLeakDetectorFactory.java @@ -118,7 +118,8 @@ private void onAfterClassCore(ITestClass testClass) { synchronized (staticLock) { instanceCountSnapshot = testClassInventory.get(testClassName); if (instanceCountSnapshot == null) { - testClassInventory.put(testClassName, instanceCountSnapshot = new AtomicInteger(0)); + throw new IllegalStateException( + "BeforeClass in TestListener was not called for testClass " + testClassName); } } From 2508e45e2c267d143e0256efd46321c1fc7bad1e Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Tue, 18 Nov 2025 20:25:02 +0100 Subject: [PATCH 65/65] Update sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdRequestDecoder.java Co-authored-by: Annie Liang <64233642+xinlian12@users.noreply.github.com> --- .../directconnectivity/rntbd/RntbdRequestDecoder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdRequestDecoder.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdRequestDecoder.java index bab67b3dd231..972ca9193d8c 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdRequestDecoder.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/rntbd/RntbdRequestDecoder.java @@ -14,7 +14,7 @@ public final class RntbdRequestDecoder extends ByteToMessageDecoder { - private static final Logger logger = LoggerFactory.getLogger(RntbdContextRequestDecoder.class); + private static final Logger logger = LoggerFactory.getLogger(RntbdRequestDecoder.class); private static final boolean leakDetectionDebuggingEnabled = ResourceLeakDetector.getLevel().ordinal() >= ResourceLeakDetector.Level.ADVANCED.ordinal();