5
5
#include < ydb/core/tablet_flat/shared_sausagecache.h>
6
6
#include < ydb/core/tx/datashard/ut_common/datashard_ut_common.h>
7
7
#include < ydb/core/tx/columnshard/common/limits.h>
8
+ #include < ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.h>
8
9
#include < ydb/core/tx/limiter/grouped_memory/usage/service.h>
9
10
#include < ydb/library/actors/testlib/test_runtime.h>
10
11
@@ -505,7 +506,7 @@ Y_UNIT_TEST(ResourceBroker_ConfigCS) {
505
506
506
507
const ui64 compactionMemoryLimitPercent = 36 ;
507
508
auto memoryControllerConfig = serverSettings.AppConfig ->MutableMemoryControllerConfig ();
508
- memoryControllerConfig->SetColumnTablesCompactionLimitPercent (compactionMemoryLimitPercent);
509
+ memoryControllerConfig->SetCompactionLimitPercent (compactionMemoryLimitPercent);
509
510
510
511
auto server = MakeIntrusive<TWithMemoryControllerServer>(serverSettings);
511
512
auto & runtime = *server->GetRuntime ();
@@ -553,8 +554,8 @@ Y_UNIT_TEST(GroupedMemoryLimiter_ConfigCS) {
553
554
const ui64 compactionMemoryLimitPercent = 36 ;
554
555
const ui64 readExecutionMemoryLimitPercent = 20 ;
555
556
auto memoryControllerConfig = serverSettings.AppConfig ->MutableMemoryControllerConfig ();
556
- memoryControllerConfig->SetColumnTablesCompactionLimitPercent (compactionMemoryLimitPercent);
557
- memoryControllerConfig->SetColumnTablesReadExecutionLimitPercent (readExecutionMemoryLimitPercent);
557
+ memoryControllerConfig->SetCompactionLimitPercent (compactionMemoryLimitPercent);
558
+ memoryControllerConfig->SetQueryExecutionLimitPercent (readExecutionMemoryLimitPercent);
558
559
559
560
ui64 currentHardMemoryLimit = 1000_MB;
560
561
auto server = MakeIntrusive<TWithMemoryControllerServer>(serverSettings);
@@ -566,29 +567,47 @@ Y_UNIT_TEST(GroupedMemoryLimiter_ConfigCS) {
566
567
auto counters = runtime.GetAppData ().Counters ;
567
568
auto compactionCounters = counters->GetSubgroup (" module_id" , " grouped_memory_limiter" )->GetSubgroup (" limiter_name" , " Comp_0" )->GetSubgroup (" stage" , " general" );
568
569
auto scanCounters = counters->GetSubgroup (" module_id" , " grouped_memory_limiter" )->GetSubgroup (" limiter_name" , " Scan_0" )->GetSubgroup (" stage" , " general" );
570
+ auto dedupCounters = counters->GetSubgroup (" module_id" , " grouped_memory_limiter" )->GetSubgroup (" limiter_name" , " Dedu_0" )->GetSubgroup (" stage" , " general" );
569
571
570
572
InitRoot (server, sender);
571
573
572
574
auto checkMemoryLimits = [&]() {
573
575
using OlapLimits = NKikimr::NOlap::TGlobalLimits;
574
576
UNIT_ASSERT_DOUBLES_EQUAL (
575
577
static_cast <double >(currentHardMemoryLimit * OlapLimits::GroupedMemoryLimiterSoftLimitCoefficient *
576
- (1.0 - ColumnTablesDeduplicationGroupedMemoryFraction) * readExecutionMemoryLimitPercent / 100 ),
578
+ (1.0 - ColumnTablesDeduplicationGroupedMemoryFraction) * readExecutionMemoryLimitPercent / 100 *
579
+ NOlap::NGroupedMemoryManager::TScanMemoryLimiterPolicy::HardLimitMultiplier),
577
580
static_cast <double >(scanCounters->GetCounter (" Value/Limit/Soft/Bytes" )->Val ()),
578
581
1_KB);
579
582
580
583
UNIT_ASSERT_DOUBLES_EQUAL (
581
- static_cast <double >(currentHardMemoryLimit * (1.0 - ColumnTablesDeduplicationGroupedMemoryFraction) * readExecutionMemoryLimitPercent / 100 ),
584
+ static_cast <double >(currentHardMemoryLimit * (1.0 - ColumnTablesDeduplicationGroupedMemoryFraction) * readExecutionMemoryLimitPercent / 100 *
585
+ NOlap::NGroupedMemoryManager::TScanMemoryLimiterPolicy::HardLimitMultiplier),
582
586
static_cast <double >(scanCounters->GetCounter (" Value/Limit/Hard/Bytes" )->Val ()),
583
587
1_KB);
584
588
585
589
UNIT_ASSERT_DOUBLES_EQUAL (
586
- static_cast <double >(currentHardMemoryLimit * OlapLimits::GroupedMemoryLimiterSoftLimitCoefficient * compactionMemoryLimitPercent / 100 ),
590
+ static_cast <double >(currentHardMemoryLimit * OlapLimits::GroupedMemoryLimiterSoftLimitCoefficient *
591
+ (1.0 - ColumnTablesDeduplicationGroupedMemoryFraction) * readExecutionMemoryLimitPercent / 100 *
592
+ NOlap::NGroupedMemoryManager::TDeduplicationMemoryLimiterPolicy::HardLimitMultiplier),
593
+ static_cast <double >(dedupCounters->GetCounter (" Value/Limit/Soft/Bytes" )->Val ()),
594
+ 1_KB);
595
+
596
+ UNIT_ASSERT_DOUBLES_EQUAL (
597
+ static_cast <double >(currentHardMemoryLimit * (1.0 - ColumnTablesDeduplicationGroupedMemoryFraction) * readExecutionMemoryLimitPercent / 100 *
598
+ NOlap::NGroupedMemoryManager::TDeduplicationMemoryLimiterPolicy::HardLimitMultiplier),
599
+ static_cast <double >(dedupCounters->GetCounter (" Value/Limit/Hard/Bytes" )->Val ()),
600
+ 1_KB);
601
+
602
+ UNIT_ASSERT_DOUBLES_EQUAL (
603
+ static_cast <double >(currentHardMemoryLimit * OlapLimits::GroupedMemoryLimiterSoftLimitCoefficient * compactionMemoryLimitPercent / 100 *
604
+ NOlap::NGroupedMemoryManager::TCompMemoryLimiterPolicy::HardLimitMultiplier),
587
605
static_cast <double >(compactionCounters->GetCounter (" Value/Limit/Soft/Bytes" )->Val ()),
588
606
1_KB);
589
607
590
608
UNIT_ASSERT_DOUBLES_EQUAL (
591
- static_cast <double >(currentHardMemoryLimit * compactionMemoryLimitPercent / 100.0 ),
609
+ static_cast <double >(currentHardMemoryLimit * compactionMemoryLimitPercent / 100.0 *
610
+ NOlap::NGroupedMemoryManager::TCompMemoryLimiterPolicy::HardLimitMultiplier),
592
611
static_cast <double >(compactionCounters->GetCounter (" Value/Limit/Hard/Bytes" )->Val ()),
593
612
1_KB);
594
613
};
@@ -608,6 +627,68 @@ Y_UNIT_TEST(GroupedMemoryLimiter_ConfigCS) {
608
627
runtime.SimulateSleep (TDuration::Seconds (2 ));
609
628
checkMemoryLimits ();
610
629
}
630
+
631
+ Y_UNIT_TEST (ColumnShardCaches_Config) {
632
+ using namespace NResourceBroker ;
633
+
634
+ TPortManager pm;
635
+ TServerSettings serverSettings (pm.GetPort (2134 ));
636
+ serverSettings.SetDomainName (" Root" ).SetUseRealThreads (false );
637
+
638
+ const ui64 sharedCacheMaxPercent = 50 ;
639
+ auto memoryControllerConfig = serverSettings.AppConfig ->MutableMemoryControllerConfig ();
640
+ memoryControllerConfig->SetSharedCacheMaxPercent (sharedCacheMaxPercent);
641
+
642
+ ui64 currentHardMemoryLimit = 1000_MB;
643
+ auto server = MakeIntrusive<TWithMemoryControllerServer>(serverSettings);
644
+ server->ProcessMemoryInfo ->CGroupLimit = currentHardMemoryLimit;
645
+ auto & runtime = *server->GetRuntime ();
646
+ TAutoPtr<IEventHandle> handle;
647
+ auto sender = runtime.AllocateEdgeActor ();
648
+
649
+ InitRoot (server, sender);
650
+ auto counters = runtime.GetAppData ().Counters ;
651
+ auto dataAccessorCache = counters->GetSubgroup (" module_id" , " general_cache" )->GetSubgroup (" cache_name" , " portions_metadata" )->GetSubgroup (" signals_owner" , " manager" );
652
+ auto columnDataCache = counters->GetSubgroup (" module_id" , " general_cache" )->GetSubgroup (" cache_name" , " column_data" )->GetSubgroup (" signals_owner" , " manager" );
653
+ auto blobCache = counters->GetSubgroup (" type" , " BLOB_CACHE" );
654
+
655
+ auto checkMemoryLimits = [&]() {
656
+ UNIT_ASSERT_DOUBLES_EQUAL (
657
+ static_cast <double >(currentHardMemoryLimit * ColumnTablesPortionsMetaDataCacheFraction * sharedCacheMaxPercent / 100.0 * ColumnTablesCachesPercentFromShared / 100.0 ),
658
+ static_cast <double >(NKikimr::NOlap::NStorageOptimizer::IOptimizerPlanner::GetPortionsCacheLimit ()),
659
+ 1_KB);
660
+
661
+ UNIT_ASSERT_DOUBLES_EQUAL (
662
+ static_cast <double >(currentHardMemoryLimit * ColumnTablesColumnTablesDataAccessorCacheFraction * sharedCacheMaxPercent / 100.0 * ColumnTablesCachesPercentFromShared / 100.0 ),
663
+ static_cast <double >(dataAccessorCache->GetCounter (" Value/Cache/SizeLimit/Bytes" )->Val ()),
664
+ 1_KB);
665
+
666
+ UNIT_ASSERT_DOUBLES_EQUAL (
667
+ static_cast <double >(currentHardMemoryLimit * ColumnTablesColumnDataCacheFraction * sharedCacheMaxPercent / 100.0 * ColumnTablesCachesPercentFromShared / 100.0 ),
668
+ static_cast <double >(columnDataCache->GetCounter (" Value/Cache/SizeLimit/Bytes" )->Val ()),
669
+ 1_KB);
670
+
671
+ UNIT_ASSERT_DOUBLES_EQUAL (
672
+ static_cast <double >(currentHardMemoryLimit * ColumnTablesBlobCacheFraction * sharedCacheMaxPercent / 100.0 * ColumnTablesCachesPercentFromShared / 100.0 ),
673
+ static_cast <double >(blobCache->GetCounter (" MaxSizeBytes" )->Val ()),
674
+ 1_KB);
675
+ };
676
+
677
+ runtime.SimulateSleep (TDuration::Seconds (2 ));
678
+ checkMemoryLimits ();
679
+
680
+ // Check memory decrease
681
+ currentHardMemoryLimit = 500_MB;
682
+ server->ProcessMemoryInfo ->CGroupLimit = currentHardMemoryLimit;
683
+ runtime.SimulateSleep (TDuration::Seconds (2 ));
684
+ checkMemoryLimits ();
685
+
686
+ // Check memory increase
687
+ currentHardMemoryLimit = 2000_MB;
688
+ server->ProcessMemoryInfo ->CGroupLimit = currentHardMemoryLimit;
689
+ runtime.SimulateSleep (TDuration::Seconds (2 ));
690
+ checkMemoryLimits ();
691
+ }
611
692
}
612
693
613
694
}
0 commit comments